Update KMS readme with vault quick start guide (#6747)

This commit adds key-rotation for SSE-S3 objects.
To execute a key-rotation a SSE-S3 client must
 - specify the `X-Amz-Server-Side-Encryption: AES256` header
   for the destination
 - The source == destination for the COPY operation.

Fixes #6754

.github/ISSUE_TEMPLATE.md

@@ -24,6 +24,10 @@
 <!--- How has this issue affected you? What are you trying to accomplish? -->
 <!--- Providing context helps us come up with a solution that is most useful in the real world -->
 
+## Regression
+<!-- Is this issue a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
 * Version used (`minio version`):

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,6 +7,10 @@
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->
 
+## Regression
+<!-- Is this PR fixing a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## How Has This Been Tested?
 <!--- Please describe in detail how you tested your changes. -->
 <!--- Include details of your testing environment, and the tests you ran to -->

.travis.yml

@@ -1,10 +1,4 @@
 go_import_path: github.com/minio/minio
-sudo: required
-
-services:
- - docker
-
-dist: trusty
 
 language: go
 
@@ -18,20 +12,30 @@ branches:
 matrix:
   include:
     - os: linux
+      dist: trusty
+      sudo: required
       env:
         - ARCH=x86_64
-      go: 1.10.1
+      go: 1.10.4
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
         - make test GOFLAGS="-timeout 15m -race -v"
+        - make verify
         - make coverage
-        - node --version
         - cd browser && yarn && yarn test && cd ..
+    - os: windows
+      env:
+        - ARCH=x86_64
+      go: 1.10.4
+      script:
+        - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
+        - for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
+        - bash buildscripts/go-coverage.sh
 
 before_install:
-- nvm install stable
+ - if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
 
 after_success:
 - bash <(curl -s https://codecov.io/bash)

Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /go/src/github.com/minio/
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      apk add --no-cache --virtual .build-deps git && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \

Dockerfile.dev

@@ -1,27 +1,20 @@
-FROM golang:1.10.1-alpine3.7
+FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
-ENV CGO_ENABLED 0
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY minio /usr/bin/
+
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
-WORKDIR /go/src/github.com/minio/
-
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
-
-COPY . /go/src/github.com/minio/minio
-
-RUN  \
-     apk add --no-cache ca-certificates curl && \
-     apk add --no-cache --virtual .build-deps git && \
+RUN \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
 
 EXPOSE 9000
 

Dockerfile.release

@@ -9,8 +9,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
 RUN \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \

Makefile

@@ -13,7 +13,7 @@ checks:
 	@(env bash $(PWD)/buildscripts/checkgopath.sh)
 
 getdeps:
-	@echo "Installing golint" && go get -u github.com/golang/lint/golint
+	@echo "Installing golint" && go get -u golang.org/x/lint/golint
 	@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
 	@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
 	@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
@@ -61,6 +61,8 @@ check: test
 test: verifiers build
 	@echo "Running unit tests"
 	@go test $(GOFLAGS) -tags kqueue ./...
+
+verify: build
 	@echo "Verifying build"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
@@ -73,6 +75,9 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
+docker: build
+	@docker build -t $(TAG) . -f Dockerfile.dev
+
 pkg-add:
 	@echo "Adding new package $(PKG)"
 	@${GOPATH}/bin/govendor add $(PKG)

README.md

@@ -17,7 +17,7 @@ docker run -p 9000:9000 minio/minio server /data
 docker pull minio/minio:edge
 docker run -p 9000:9000 minio/minio:edge server /data
 ```
-Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
+Note: Docker will not display the autogenerated keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use autogenerated keys with containers. Please visit Minio Docker quickstart guide for more information [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
 ### Homebrew

appveyor.yml

@@ -1,55 +0,0 @@
-# version format
-version: "{build}"
-
-# Operating system (build VM template)
-os: Windows Server 2012 R2
-
-# Platform.
-platform: x64
-
-clone_folder: c:\gopath\src\github.com\minio\minio
-
-# Environment variables
-environment:
-  GOPATH: c:\gopath
-  GOROOT: c:\go
-
-# scripts that run after cloning repository
-install:
-  - set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
-  - go version
-  - go env
-  - python --version
-
-# To run your custom scripts instead of automatic MSBuild
-build_script:
-  # Compile
-  # We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
-  - ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
-  - appveyor AddCompilationMessage "Starting Compile"
-  - cd c:\gopath\src\github.com\minio\minio
-  - go run buildscripts/gen-ldflags.go > temp.txt
-  - set /p BUILD_LDFLAGS=<temp.txt
-  - go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
-  - appveyor AddCompilationMessage "Compile Success"
-
-# To run your custom scripts instead of automatic tests
-test_script:
-  # Unit tests
-  - ps: Add-AppveyorTest "Unit Tests" -Outcome Running
-  - mkdir build\coverage
-  - for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
-  - go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
-  - ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
-
-after_test:
-  - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
-  - ps: Push-AppveyorArtifact build\coverage\coverage.txt
-  - ps: Push-AppveyorArtifact build\coverage\coverage.html
-  # Upload coverage report.
-  - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
-  - pip install codecov
-  - codecov -X gcov -f "build\coverage\coverage.txt"
-
-# to disable deployment
-deploy: off

browser/app/js/alert/actions.js

@@ -22,7 +22,7 @@ export let alertId = 0
 export const set = alert => {
   const id = alertId++
   return (dispatch, getState) => {
-    if (alert.type !== "danger") {
+    if (alert.type !== "danger" || alert.autoClear) {
       setTimeout(() => {
         dispatch({
           type: CLEAR,

browser/app/js/buckets/BucketList.js

@@ -67,8 +67,12 @@ const mapDispatchToProps = dispatch => {
   return {
     fetchBuckets: () => dispatch(actionsBuckets.fetchBuckets()),
     setBucketList: buckets => dispatch(actionsBuckets.setList(buckets)),
-    selectBucket: bucket => dispatch(actionsBuckets.selectBucket(bucket))
+    selectBucket: (bucket, prefix) =>
+      dispatch(actionsBuckets.selectBucket(bucket, prefix))
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(BucketList)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(BucketList)

browser/app/js/buckets/Policy.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from '../constants'
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from '../constants'
 
 import React from "react"
 import { connect } from "react-redux"
@@ -42,37 +42,40 @@ export class Policy extends React.Component {
   render() {
     const {policy, prefix} = this.props
     let newPrefix = prefix
-
     if (newPrefix === '')
       newPrefix = '*'
 
-    return (
-      <div className="pmb-list">
-        <div className="pmbl-item">
-          { newPrefix }
+    if (policy === NONE) {
+      return <noscript />
+    } else {
+      return (
+        <div className="pmb-list">
+          <div className="pmbl-item">
+            { newPrefix }
+          </div>
+          <div className="pmbl-item">
+            <select className="form-control"
+              disabled
+              value={ policy }>
+              <option value={ READ_ONLY }>
+                Read Only
+              </option>
+              <option value={ WRITE_ONLY }>
+                Write Only
+              </option>
+              <option value={ READ_WRITE }>
+                Read and Write
+              </option>
+            </select>
+          </div>
+          <div className="pmbl-item">
+            <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
+              Remove
+            </button>
+          </div>
         </div>
-        <div className="pmbl-item">
-          <select className="form-control"
-            disabled
-            value={ policy }>
-            <option value={ READ_ONLY }>
-              Read Only
-            </option>
-            <option value={ WRITE_ONLY }>
-              Write Only
-            </option>
-            <option value={ READ_WRITE }>
-              Read and Write
-            </option>
-          </select>
-        </div>
-        <div className="pmbl-item">
-          <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
-            Remove
-          </button>
-        </div>
-      </div>
-    )
+      )
+    }
   }
 }
 

browser/app/js/buckets/__tests__/Policy.test.js

@@ -17,7 +17,7 @@
 import React from "react"
 import { shallow, mount } from "enzyme"
 import { Policy } from "../Policy"
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from "../../constants"
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from "../../constants"
 import web from "../../web"
 
 jest.mock("../../web", () => ({
@@ -31,6 +31,11 @@ describe("Policy", () => {
     shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={READ_ONLY} />)
   })
 
+  it("should not render when policy is listed as 'none'", () => {
+    const wrapper = shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={NONE} />)
+    expect(wrapper.find(".pmb-list").length).toBe(0)
+  })
+
   it("should call web.setBucketPolicy and fetchPolicies on submit", () => {
     const fetchPolicies = jest.fn()
     const wrapper = shallow(

browser/app/js/constants.js

@@ -23,6 +23,7 @@ export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
 export const READ_ONLY = "readonly"
 export const WRITE_ONLY = "writeonly"
 export const READ_WRITE = "readwrite"
+export const NONE = "none"
 
 export const SHARE_OBJECT_EXPIRY_DAYS = 5
 export const SHARE_OBJECT_EXPIRY_HOURS = 0

browser/app/js/objects/__tests__/actions.test.js

@@ -19,16 +19,27 @@ import thunk from "redux-thunk"
 import * as actionsObjects from "../actions"
 import * as alertActions from "../../alert/actions"
 import { minioBrowserPrefix } from "../../constants"
+import history from "../../history"
 
 jest.mock("../../web", () => ({
-  LoggedIn: jest.fn(() => true).mockReturnValueOnce(false),
-  ListObjects: jest.fn(() => {
-    return Promise.resolve({
-      objects: [{ name: "test1" }, { name: "test2" }],
-      istruncated: false,
-      nextmarker: "test2",
-      writable: false
-    })
+  LoggedIn: jest
+    .fn(() => true)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(false)
+    .mockReturnValueOnce(false),
+  ListObjects: jest.fn(({ bucketName }) => {
+    if (bucketName === "test-deny") {
+      return Promise.reject({
+        message: "listobjects is denied"
+      })
+    } else {
+      return Promise.resolve({
+        objects: [{ name: "test1" }, { name: "test2" }],
+        istruncated: false,
+        nextmarker: "test2",
+        writable: false
+      })
+    }
   }),
   RemoveObject: jest.fn(({ bucketName, objects }) => {
     if (!bucketName) {
@@ -160,6 +171,41 @@ describe("Objects actions", () => {
     })
   })
 
+  it("creates objects/RESET_LIST after failing to fetch the objects from bucket with ListObjects denied for LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    const expectedActions = [
+      {
+        type: "alert/SET",
+        alert: {
+          type: "danger",
+          message: "listobjects is denied",
+          id: alertActions.alertId,
+          autoClear: true
+        }
+      },
+      {
+        type: "object/RESET_LIST"
+      }
+    ]
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      const actions = store.getActions()
+      expect(actions).toEqual(expectedActions)
+    })
+  })
+
+  it("redirect to login after failing to fetch the objects from bucket for non-LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      expect(history.location.pathname.endsWith("/login")).toBeTruthy()
+    })
+  })
+
   it("creates objects/SET_SORT_BY and objects/SET_SORT_ORDER when sortObjects is called", () => {
     const store = mockStore({
       objects: {
@@ -244,7 +290,11 @@ describe("Objects actions", () => {
     const expectedActions = [
       {
         type: "alert/SET",
-        alert: { type: "danger", message: "Invalid bucket", id: 0 }
+        alert: {
+          type: "danger",
+          message: "Invalid bucket",
+          id: alertActions.alertId
+        }
       }
     ]
     return store.dispatch(actionsObjects.deleteObject("obj1")).then(() => {

browser/app/js/objects/actions.js

@@ -24,9 +24,11 @@ import {
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
 import * as alertActions from "../alert/actions"
+import * as bucketActions from "../buckets/actions"
 import { minioBrowserPrefix } from "../constants"
 
 export const SET_LIST = "objects/SET_LIST"
+export const RESET_LIST = "object/RESET_LIST"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -45,6 +47,10 @@ export const setList = (objects, marker, isTruncated) => ({
   isTruncated
 })
 
+export const resetList = () => ({
+  type: RESET_LIST
+})
+
 export const appendList = (objects, marker, isTruncated) => ({
   type: APPEND_LIST,
   objects,
@@ -60,35 +66,45 @@ export const fetchObjects = append => {
     } = getState()
     if (currentBucket) {
       return web
-      .ListObjects({
-        bucketName: currentBucket,
-        prefix: currentPrefix,
-        marker: append ? marker : ""
-      })
-      .then(res => {
-        let objects = []
-        if (res.objects) {
-          objects = res.objects.map(object => {
-            return {
-              ...object,
-              name: object.name.replace(currentPrefix, "")
-            }
-          })
-        }
-        if (append) {
-          dispatch(appendList(objects, res.nextmarker, res.istruncated))
-        } else {
-          dispatch(setList(objects, res.nextmarker, res.istruncated))
-          dispatch(setSortBy(""))
-          dispatch(setSortOrder(false))
-        }
-        dispatch(setPrefixWritable(res.writable))
-      })
-      .catch(err => {
-        dispatch(alertActions.set({ type: "danger", message: err.message }))
-        history.push("/login")
-      })
-    } 
+        .ListObjects({
+          bucketName: currentBucket,
+          prefix: currentPrefix,
+          marker: append ? marker : ""
+        })
+        .then(res => {
+          let objects = []
+          if (res.objects) {
+            objects = res.objects.map(object => {
+              return {
+                ...object,
+                name: object.name.replace(currentPrefix, "")
+              }
+            })
+          }
+          if (append) {
+            dispatch(appendList(objects, res.nextmarker, res.istruncated))
+          } else {
+            dispatch(setList(objects, res.nextmarker, res.istruncated))
+            dispatch(setSortBy(""))
+            dispatch(setSortOrder(false))
+          }
+          dispatch(setPrefixWritable(res.writable))
+        })
+        .catch(err => {
+          if (web.LoggedIn()) {
+            dispatch(
+              alertActions.set({
+                type: "danger",
+                message: err.message,
+                autoClear: true
+              })
+            )
+            dispatch(resetList())
+          } else {
+            history.push("/login")
+          }
+        })
+    }
   }
 }
 

browser/app/js/objects/reducer.js

@@ -50,6 +50,13 @@ export default (
         marker: action.marker,
         isTruncated: action.isTruncated
       }
+    case actionsObjects.RESET_LIST:
+      return {
+        ...state,
+        list: [],
+        marker: "",
+        isTruncated: false
+      }
     case actionsObjects.APPEND_LIST:
       return {
         ...state,

browser/app/less/inc/list.less

@@ -109,7 +109,15 @@ div.fesl-row {
     /*--------------------------
         Icons
     ----------------------------*/
-    &[data-type=folder] { .list-type(#a1d6dd, '\f114'); }
+    &[data-type=folder] {
+        .list-type(#a1d6dd, '\f114');
+
+        .fesl-item-name {
+            a {
+                cursor: pointer;
+            }
+        }
+    }
     &[data-type=pdf] {.list-type(#fa7775, '\f1c1'); }
     &[data-type=zip] { .list-type(#427089, '\f1c6'); }
     &[data-type=audio] { .list-type(#009688, '\f1c7'); }

buildscripts/go-coverage.sh

@@ -4,7 +4,7 @@ set -e
 echo "" > coverage.txt
 
 for d in $(go list ./... | grep -v browser); do
-    go test -coverprofile=profile.out -covermode=atomic "$d"
+    go test -v -coverprofile=profile.out -covermode=atomic "$d"
     if [ -f profile.out ]; then
         cat profile.out >> coverage.txt
         rm profile.out

cmd/acl-handlers.go

@@ -21,6 +21,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -56,6 +57,8 @@ type accessControlPolicy struct {
 func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketACL")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -103,6 +106,8 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
 func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectACL")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	object := vars["object"]
@@ -121,7 +126,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if object exists.
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object)
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return

cmd/admin-handlers_test.go

@@ -27,6 +27,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -37,160 +38,184 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "28",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "",
-	"browser": "on",
-	"worm": "off",
-	"domain": "",
-	"storageclass": {
-		"standard": "",
-		"rrs": ""
-	},
-	"cache": {
-		"drives": [],
-		"expiry": 90,
-		"maxuse": 80,
-		"exclude": []
-	},
-	"kms": {
-		"vault": {
-			"endpoint": "",
-			"auth": {
-				"type": "",
-				"approle": {
-					"id": "",
-					"secret": ""
-				}
-			},
-			"key-id": {
-				"name": "",
-				"version": 0
-			}
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"deliveryMode": 0,
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"url": "",
-				"index": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"mqtt": {
-			"1": {
-				"enable": false,
-				"broker": "",
-				"topic": "",
-				"qos": 0,
-				"clientId": "",
-				"username": "",
-				"password": "",
-				"reconnectInterval": 0,
-				"keepAliveInterval": 0
-			}
-		},
-		"mysql": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"dsnString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"clientID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"format": "",
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	    },
-	    "logger": {
-		"console": {
-		    "enabled": true
-		},
-		"http": {
-		    "1": {
-			"enabled": false,
-			"endpoint": "http://user:example@localhost:9001/api/endpoint"
-		    }
-		}
-	    }
-
-	}`)
+  "version": "31",
+  "credential": {
+    "accessKey": "minio",
+    "secretKey": "minio123"
+  },
+  "region": "us-east-1",
+  "worm": "off",
+  "storageclass": {
+    "standard": "",
+    "rrs": ""
+  },
+  "cache": {
+    "drives": [],
+    "expiry": 90,
+    "maxuse": 80,
+    "exclude": []
+  },
+  "kms": {
+    "vault": {
+      "endpoint": "",
+      "auth": {
+        "type": "",
+        "approle": {
+          "id": "",
+          "secret": ""
+        }
+      },
+      "key-id": {
+        "name": "",
+        "version": 0
+      }
+    }
+  },
+  "notify": {
+    "amqp": {
+      "1": {
+        "enable": false,
+        "url": "",
+        "exchange": "",
+        "routingKey": "",
+        "exchangeType": "",
+        "deliveryMode": 0,
+        "mandatory": false,
+        "immediate": false,
+        "durable": false,
+        "internal": false,
+        "noWait": false,
+        "autoDeleted": false
+      }
+    },
+    "elasticsearch": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "url": "",
+        "index": ""
+      }
+    },
+    "kafka": {
+      "1": {
+        "enable": false,
+        "brokers": null,
+        "topic": "",
+        "tls": {
+          "enable": false,
+          "skipVerify": false,
+          "clientAuth": 0
+        },
+        "sasl": {
+          "enable": false,
+          "username": "",
+          "password": ""
+        }
+      }
+    },
+    "mqtt": {
+      "1": {
+        "enable": false,
+        "broker": "",
+        "topic": "",
+        "qos": 0,
+        "clientId": "",
+        "username": "",
+        "password": "",
+        "reconnectInterval": 0,
+        "keepAliveInterval": 0
+      }
+    },
+    "mysql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "dsnString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "nats": {
+      "1": {
+        "enable": false,
+        "address": "",
+        "subject": "",
+        "username": "",
+        "password": "",
+        "token": "",
+        "secure": false,
+        "pingInterval": 0,
+        "streaming": {
+          "enable": false,
+          "clusterID": "",
+          "clientID": "",
+          "async": false,
+          "maxPubAcksInflight": 0
+        }
+      }
+    },
+    "postgresql": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "connectionString": "",
+        "table": "",
+        "host": "",
+        "port": "",
+        "user": "",
+        "password": "",
+        "database": ""
+      }
+    },
+    "redis": {
+      "1": {
+        "enable": false,
+        "format": "namespace",
+        "address": "",
+        "password": "",
+        "key": ""
+      }
+    },
+    "webhook": {
+      "1": {
+        "enable": false,
+        "endpoint": ""
+      }
+    }
+  },
+  "logger": {
+    "console": {
+      "enabled": true
+    },
+    "http": {
+      "1": {
+        "enabled": false,
+        "endpoint": "https://username:password@example.com/api"
+      }
+    }
+  },
+  "compress": {
+    "enabled": false,
+    "extensions":[".txt",".log",".csv",".json"],
+    "mime-types":["text/csv","text/plain","application/json"]
+  },
+  "openid": {
+    "jwks": {
+      "url": ""
+    }
+  },
+  "policy": {
+    "opa": {
+      "url": "",
+      "authToken": ""
+    }
+  }
+}
+`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for
@@ -274,7 +299,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 			objectName := fmt.Sprintf("%s-%d", objName, i)
 			_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
 				mustGetHashReader(t, bytes.NewReader([]byte("hello")),
-					int64(len("hello")), "", ""), nil)
+					int64(len("hello")), "", ""), nil, ObjectOptions{})
 			if err != nil {
 				t.Fatalf("Failed to create %s - %v", objectName,
 					err)
@@ -286,14 +311,14 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 	{
 		objName := "mpObject"
 		uploadID, err := atb.objLayer.NewMultipartUpload(context.Background(), bucketName,
-			objName, nil)
+			objName, nil, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp new error: %v", err)
 		}
 
 		_, err = atb.objLayer.PutObjectPart(context.Background(), bucketName, objName,
 			uploadID, 3, mustGetHashReader(t, bytes.NewReader(
-				[]byte("hello")), int64(len("hello")), "", ""))
+				[]byte("hello")), int64(len("hello")), "", ""), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp put error: %v", err)
 		}
@@ -472,6 +497,8 @@ func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*htt
 
 	// Set body
 	req.Body = ioutil.NopCloser(bytes.NewReader(body))
+	req.ContentLength = int64(len(body))
+
 	// Set sha-sum header
 	req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body))
 
@@ -498,10 +525,16 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
+	var wg sync.WaitGroup
+
 	// Setting up a go routine to simulate ServerRouter's
 	// handleServiceSignals for stop and restart commands.
 	if cmd == restartCmd {
-		go testServiceSignalReceiver(cmd, t)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			testServiceSignalReceiver(cmd, t)
+		}()
 	}
 	credentials := globalServerConfig.GetCredential()
 
@@ -537,6 +570,9 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
 			http.StatusOK, rec.Code, string(resp))
 	}
+
+	// Wait until testServiceSignalReceiver() called in a goroutine quits.
+	wg.Wait()
 }
 
 // Test for service status management REST API.
@@ -593,7 +629,7 @@ func TestServiceSetCreds(t *testing.T) {
 			t.Fatalf("JSONify err: %v", err)
 		}
 
-		ebody, err := madmin.EncryptServerConfigData(credentials.SecretKey, body)
+		ebody, err := madmin.EncryptData(credentials.SecretKey, body)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -691,16 +727,12 @@ func TestSetConfigHandler(t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	// SetConfigHandler restarts minio setup - need to start a
-	// signal receiver to receive on globalServiceSignalCh.
-	go testServiceSignalReceiver(restartCmd, t)
-
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("config", "")
 
 	password := globalServerConfig.GetCredential().SecretKey
-	econfigJSON, err := madmin.EncryptServerConfigData(password, configJSON)
+	econfigJSON, err := madmin.EncryptData(password, configJSON)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -720,7 +752,7 @@ func TestSetConfigHandler(t *testing.T) {
 	// Check that a very large config file returns an error.
 	{
 		// Make a large enough config string
-		invalidCfg := []byte(strings.Repeat("A", maxConfigJSONSize+1))
+		invalidCfg := []byte(strings.Repeat("A", maxEConfigJSONSize+1))
 		req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
 			int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
 		if err != nil {
@@ -750,7 +782,7 @@ func TestSetConfigHandler(t *testing.T) {
 		adminTestBed.router.ServeHTTP(rec, req)
 		respBody := string(rec.Body.Bytes())
 		if rec.Code != http.StatusBadRequest ||
-			!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
+			!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
 			t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
 		}
 	}

cmd/admin-heal-ops.go

@@ -20,12 +20,15 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net/http"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // healStatusSummary - overall short summary of a healing sequence
@@ -121,6 +124,35 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
 	return h, exists
 }
 
+func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
+	var hsp madmin.HealStopSuccess
+	he, exists := ahs.getHealSequence(path)
+	if !exists {
+		hsp = madmin.HealStopSuccess{
+			ClientToken: "invalid",
+			StartTime:   UTCNow(),
+		}
+	} else {
+		hsp = madmin.HealStopSuccess{
+			ClientToken:   he.clientToken,
+			ClientAddress: he.clientAddress,
+			StartTime:     he.startTime,
+		}
+
+		he.stop()
+		for !he.hasEnded() {
+			time.Sleep(1 * time.Second)
+		}
+		ahs.Lock()
+		defer ahs.Unlock()
+		// Heal sequence explicitly stopped, remove it.
+		delete(ahs.healSeqMap, path)
+	}
+
+	b, err := json.Marshal(&hsp)
+	return b, toAdminAPIErrCode(err)
+}
+
 // LaunchNewHealSequence - launches a background routine that performs
 // healing according to the healSequence argument. For each heal
 // sequence, state is stored in the `globalAllHealState`, which is a
@@ -141,20 +173,20 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 			existsAndLive = true
 		}
 	}
+
 	if existsAndLive {
 		// A heal sequence exists on the given path.
 		if h.forceStarted {
-			// stop the running heal sequence - wait for
-			// it to finish.
+			// stop the running heal sequence - wait for it to finish.
 			he.stop()
 			for !he.hasEnded() {
-				time.Sleep(10 * time.Second)
+				time.Sleep(1 * time.Second)
 			}
 		} else {
 			errMsg = "Heal is already running on the given path " +
 				"(use force-start option to stop and start afresh). " +
-				fmt.Sprintf("The heal was started by IP %s at %s",
-					h.clientAddress, h.startTime)
+				fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
+					h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
 
 			return nil, ErrHealAlreadyRunning, errMsg
 		}
@@ -283,7 +315,7 @@ type healSequence struct {
 	// bucket, and prefix on which heal seq. was initiated
 	bucket, objPrefix string
 
-	// path is just bucket + "/" + objPrefix
+	// path is just pathJoin(bucket, objPrefix)
 	path string
 
 	// time at which heal sequence was started
@@ -328,7 +360,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
 	return &healSequence{
 		bucket:        bucket,
 		objPrefix:     objPrefix,
-		path:          bucket + "/" + objPrefix,
+		path:          pathJoin(bucket, objPrefix),
 		startTime:     UTCNow(),
 		clientToken:   mustGetUUID(),
 		clientAddress: clientAddr,
@@ -383,7 +415,6 @@ func (h *healSequence) stop() {
 // sequence automatically resumes. The return value indicates if the
 // operation succeeded.
 func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
-
 	// start a timer to keep an upper time limit to find an empty
 	// slot to add the given heal result - if no slot is found it
 	// means that the server is holding the maximum amount of
@@ -519,6 +550,9 @@ func (h *healSequence) traverseAndHeal() {
 	// Start with format healing
 	checkErr(h.healDiskFormat)
 
+	// Start healing the config.
+	checkErr(h.healConfig)
+
 	// Heal buckets and objects
 	checkErr(h.healBuckets)
 
@@ -529,9 +563,72 @@ func (h *healSequence) traverseAndHeal() {
 	close(h.traverseAndHealDoneCh)
 }
 
+// healConfig - heals config.json, retrun value indicates if a failure occurred.
+func (h *healSequence) healConfig() error {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// NOTE: Healing on configs is run regardless
+	// of any bucket being selected, this is to ensure that
+	// configs are always uptodate and correct.
+	marker := ""
+	isTruncated := true
+	for isTruncated {
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to heal
+			waitCount := 60
+			// Any requests in progress, delay the heal.
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
+		}
+
+		// Lists all objects under `config` prefix.
+		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, minioConfigPrefix,
+			marker, "", 1000)
+		if err != nil {
+			return errFnHealFromAPIErr(err)
+		}
+
+		for index := range objectInfos.Objects {
+			if h.isQuitting() {
+				return errHealStopSignalled
+			}
+			o := objectInfos.Objects[index]
+			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun)
+			// Object might have been deleted, by the time heal
+			// was attempted we ignore this file an move on.
+			if isErrObjectNotFound(herr) {
+				continue
+			}
+			if herr != nil {
+				return herr
+			}
+			res.Type = madmin.HealItemBucketMetadata
+			if err = h.pushHealResultItem(res); err != nil {
+				return err
+			}
+		}
+
+		isTruncated = objectInfos.IsTruncated
+		marker = objectInfos.NextMarker
+	}
+
+	return nil
+}
+
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func (h *healSequence) healDiskFormat() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -557,6 +654,10 @@ func (h *healSequence) healDiskFormat() error {
 
 // healBuckets - check for all buckets heal or just particular bucket.
 func (h *healSequence) healBuckets() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
 		return h.healBucket(h.bucket)
@@ -584,10 +685,6 @@ func (h *healSequence) healBuckets() error {
 
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string) error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -610,10 +707,9 @@ func (h *healSequence) healBucket(bucket string) error {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix)
+			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
 			if err == nil {
-				err = h.healObject(bucket, h.objPrefix)
-				if err != nil {
+				if err = h.healObject(bucket, h.objPrefix); err != nil {
 					return err
 				}
 			}
@@ -622,17 +718,40 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
+	entries := runtime.NumCPU() * globalEndpoints.Nodes()
+
 	marker := ""
 	isTruncated := true
 	for isTruncated {
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to heal
+			waitCount := 60
+			// Any requests in progress, delay the heal.
+			for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
+		}
+
+		// Heal numCPU * nodes objects at a time.
 		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
-			h.objPrefix, marker, "", 1000)
+			h.objPrefix, marker, "", entries)
 		if err != nil {
 			return errFnHealFromAPIErr(err)
 		}
 
-		for _, o := range objectInfos.Objects {
-			if err := h.healObject(o.Bucket, o.Name); err != nil {
+		g := errgroup.WithNErrs(len(objectInfos.Objects))
+		for index := range objectInfos.Objects {
+			index := index
+			g.Go(func() error {
+				o := objectInfos.Objects[index]
+				return h.healObject(o.Bucket, o.Name)
+			}, index)
+		}
+
+		for _, err := range g.Wait() {
+			if err != nil {
 				return err
 			}
 		}
@@ -656,6 +775,9 @@ func (h *healSequence) healObject(bucket, object string) error {
 	}
 
 	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	if isErrObjectNotFound(err) {
+		return nil
+	}
 	if err != nil {
 		hri.Detail = err.Error()
 	}

cmd/admin-router.go

@@ -53,19 +53,58 @@ func registerAdminRouter(router *mux.Router) {
 	// Info operations
 	adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
 
-	/// Heal operations
+	if globalIsDistXL || globalIsXL {
+		/// Heal operations
 
-	// Heal processing endpoint.
-	adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		// Heal processing endpoint.
+		adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+		adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+	}
+
+	// Profiling operations
+	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+		Queries("profilerType", "{profilerType:.*}")
+	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
 
 	/// Config operations
 
 	// Update credentials
-	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateCredentialsHandler))
+	adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
 	// Get config
 	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 	// Set config
 	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+
+	// Get config keys/values
+	adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
+	// Set config keys/values
+	adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
+
+	// -- IAM APIs --
+
+	// Add policy IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Add user IAM
+	adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
+		Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
+	adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
+		Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+
+	// Remove policy IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+
+	// Remove user IAM
+	adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+
+	// List users
+	adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+
+	// List policies
+	adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+
+	// If none of the routes match.
+	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
 }

cmd/admin-rpc-client.go

@@ -68,6 +68,22 @@ func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
 	return reply, err
 }
 
+// StartProfiling - starts profiling in the remote server.
+func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
+	args := StartProfilingArgs{Profiler: profiler}
+	reply := VoidReply{}
+	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
+}
+
+// DownloadProfilingData - returns profiling data of the remote server.
+func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+
+	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
+	return reply, err
+}
+
 // NewAdminRPCClient - returns new admin RPC client.
 func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
 	scheme := "http"
@@ -112,6 +128,8 @@ type adminCmdRunner interface {
 	ReInitFormat(dryRun bool) error
 	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
+	StartProfiling(string) error
+	DownloadProfilingData() ([]byte, error)
 }
 
 // adminPeer - represents an entity that implements admin API RPCs.
@@ -139,9 +157,9 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
+		logger.FatalIf(err, "Unable to parse Admin RPC Host")
 		rpcClient, err := NewAdminRPCClient(host)
-		logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client")
 		adminPeerList = append(adminPeerList, adminPeer{
 			addr:      hostStr,
 			cmdRunner: rpcClient,

cmd/admin-rpc-server.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"path"
 
 	"github.com/gorilla/mux"
@@ -52,6 +51,23 @@ func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoDa
 	return err
 }
 
+// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
+type StartProfilingArgs struct {
+	AuthArgs
+	Profiler string
+}
+
+// StartProfiling - starts profiling of this server
+func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
+	return receiver.local.StartProfiling(args.Profiler)
+}
+
+// DownloadProfilingData - stops and returns profiling data of this server
+func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
+	*reply, err = receiver.local.DownloadProfilingData()
+	return
+}
+
 // GetConfig - returns the config.json of this server.
 func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
 	*reply, err = receiver.local.GetConfig()
@@ -81,7 +97,7 @@ func NewAdminRPCServer() (*xrpc.Server, error) {
 // registerAdminRPCRouter - creates and registers Admin RPC server and its router.
 func registerAdminRPCRouter(router *mux.Router) {
 	rpcServer, err := NewAdminRPCServer()
-	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(adminServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
 }

cmd/api-datatypes.go

@@ -25,87 +25,6 @@ const (
 	responseRequestIDKey = "x-amz-request-id"
 )
 
-// CSVFileHeaderInfo -Can be either USE IGNORE OR NONE, defines what to do with
-// the first row
-type CSVFileHeaderInfo string
-
-// Constants for file header info.
-const (
-	CSVFileHeaderInfoNone   CSVFileHeaderInfo = "NONE"
-	CSVFileHeaderInfoIgnore                   = "IGNORE"
-	CSVFileHeaderInfoUse                      = "USE"
-)
-
-// SelectCompressionType - ONLY GZIP is supported
-type SelectCompressionType string
-
-// Constants for compression types under select API.
-const (
-	SelectCompressionNONE SelectCompressionType = "NONE"
-	SelectCompressionGZIP                       = "GZIP"
-	SelectCompressionBZIP                       = "BZIP2"
-)
-
-// CSVQuoteFields - Can be either Always or AsNeeded
-type CSVQuoteFields string
-
-// Constants for csv quote styles.
-const (
-	CSVQuoteFieldsAlways   CSVQuoteFields = "Always"
-	CSVQuoteFieldsAsNeeded                = "AsNeeded"
-)
-
-// QueryExpressionType - Currently can only be SQL
-type QueryExpressionType string
-
-// Constants for expression type.
-const (
-	QueryExpressionTypeSQL QueryExpressionType = "SQL"
-)
-
-// JSONType determines json input serialization type.
-type JSONType string
-
-// Constants for JSONTypes.
-const (
-	JSONDocumentType JSONType = "Document"
-	JSONStreamType            = "Stream"
-	JSONLinesType             = "Lines"
-)
-
-// ObjectSelectRequest - represents the input select body
-type ObjectSelectRequest struct {
-	XMLName            xml.Name `xml:"SelectObjectContentRequest" json:"-"`
-	Expression         string
-	ExpressionType     QueryExpressionType
-	InputSerialization struct {
-		CompressionType SelectCompressionType
-		CSV             *struct {
-			FileHeaderInfo       CSVFileHeaderInfo
-			RecordDelimiter      string
-			FieldDelimiter       string
-			QuoteCharacter       string
-			QuoteEscapeCharacter string
-			Comments             string
-		}
-		JSON *struct {
-			Type JSONType
-		}
-	}
-	OutputSerialization struct {
-		CSV *struct {
-			QuoteFields          CSVQuoteFields
-			RecordDelimiter      string
-			FieldDelimiter       string
-			QuoteCharacter       string
-			QuoteEscapeCharacter string
-		}
-		JSON *struct {
-			RecordDelimiter string
-		}
-	}
-}
-
 // ObjectIdentifier carries key name for the object to delete.
 type ObjectIdentifier struct {
 	ObjectName string `xml:"Key"`

cmd/api-errors.go

@@ -187,11 +187,15 @@ const (
 	// new error codes here.
 
 	ErrMalformedJSON
+	ErrAdminNoSuchUser
+	ErrAdminNoSuchPolicy
+	ErrAdminInvalidArgument
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
 	ErrAdminConfigNoQuorum
 	ErrAdminConfigTooLarge
 	ErrAdminConfigBadJSON
+	ErrAdminConfigDuplicateKeys
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
@@ -249,7 +253,6 @@ const (
 	ErrParseUnsupportedAlias
 	ErrParseUnsupportedSyntax
 	ErrParseUnknownOperator
-	ErrParseInvalidPathComponent
 	ErrParseMissingIdentAfterAt
 	ErrParseUnexpectedOperator
 	ErrParseUnexpectedTerm
@@ -288,8 +291,10 @@ const (
 	ErrEvaluatorInvalidTimestampFormatPatternToken
 	ErrEvaluatorInvalidTimestampFormatPatternSymbol
 	ErrEvaluatorBindingDoesNotExist
-	ErrInvalidColumnIndex
 	ErrMissingHeaders
+	ErrAdminConfigNotificationTargetsFailed
+	ErrAdminProfilerNotEnabled
+	ErrInvalidDecompressedSize
 )
 
 // error code to APIError structure, these fields carry respective
@@ -860,6 +865,21 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The JSON you provided was not well-formed or did not validate against our published format.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoSuchUser: {
+		Code:           "XMinioAdminNoSuchUser",
+		Description:    "The specified user does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminNoSuchPolicy: {
+		Code:           "XMinioAdminNoSuchPolicy",
+		Description:    "The canned policy does not exist.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
+	ErrAdminInvalidArgument: {
+		Code:           "XMinioAdminInvalidArgument",
+		Description:    "Invalid arguments specified.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminInvalidAccessKey: {
 		Code:           "XMinioAdminInvalidAccessKey",
 		Description:    "The access key is invalid.",
@@ -878,14 +898,29 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 	ErrAdminConfigTooLarge: {
 		Code: "XMinioAdminConfigTooLarge",
 		Description: fmt.Sprintf("Configuration data provided exceeds the allowed maximum of %d bytes",
-			maxConfigJSONSize),
+			maxEConfigJSONSize),
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAdminConfigBadJSON: {
 		Code:           "XMinioAdminConfigBadJSON",
+		Description:    "JSON configuration provided is of incorrect format",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminConfigDuplicateKeys: {
+		Code:           "XMinioAdminConfigDuplicateKeys",
 		Description:    "JSON configuration provided has objects with duplicate keys",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigNotificationTargetsFailed: {
+		Code:           "XMinioAdminNotificationTargetsTestFailed",
+		Description:    "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminProfilerNotEnabled: {
+		Code:           "XMinioAdminProfilerNotEnabled",
+		Description:    "Unable to perform the requested operation because profiling is not enabled",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminCredentialsMismatch: {
 		Code:           "XMinioAdminCredentialsMismatch",
 		Description:    "Credentials in config mismatch with server environment variables",
@@ -1186,11 +1221,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "The SQL expression contains an invalid operator.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrParseInvalidPathComponent: {
-		Code:           "ParseInvalidPathComponent",
-		Description:    "The SQL expression contains an invalid path component.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrParseMissingIdentAfterAt: {
 		Code:           "ParseMissingIdentAfterAt",
 		Description:    "Did not find the expected identifier after the @ symbol in the SQL expression.",
@@ -1376,11 +1406,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Time stamp format pattern contains an invalid symbol in the SQL expression.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrInvalidColumnIndex: {
-		Code:           "InvalidColumnIndex",
-		Description:    "Column index in the SQL expression is invalid.",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrEvaluatorBindingDoesNotExist: {
 		Code:           "ErrEvaluatorBindingDoesNotExist",
 		Description:    "A column name or a path provided does not exist in the SQL expression",
@@ -1391,6 +1416,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Some headers in the query are missing from the file. Check the file and try again.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidDecompressedSize: {
+		Code:           "XMinioInvalidDecompressedSize",
+		Description:    "The data provided is unfit for decompression",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -1404,8 +1434,16 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 	// Verify if the underlying error is signature mismatch.
 	switch err {
+	case errInvalidArgument:
+		apiErr = ErrAdminInvalidArgument
+	case errNoSuchUser:
+		apiErr = ErrAdminNoSuchUser
+	case errNoSuchPolicy:
+		apiErr = ErrAdminNoSuchPolicy
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
+	case errInvalidRange:
+		apiErr = ErrInvalidRange
 	case errDataTooLarge:
 		apiErr = ErrEntityTooLarge
 	case errDataTooSmall:
@@ -1415,10 +1453,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
 	// SSE errors
+	case errInvalidEncryptionParameters:
+		apiErr = ErrInvalidEncryptionParameters
 	case crypto.ErrInvalidEncryptionMethod:
 		apiErr = ErrInvalidEncryptionMethod
-	case errInsecureSSERequest:
-		apiErr = ErrInsecureSSECustomerRequest
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
 	case crypto.ErrInvalidCustomerKey:
@@ -1435,7 +1473,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrSSEEncryptedObject
 	case errInvalidSSEParameters:
 		apiErr = ErrInvalidSSECustomerParameters
-	case crypto.ErrInvalidCustomerKey:
+	case crypto.ErrInvalidCustomerKey, crypto.ErrSecretKeyMismatch:
 		apiErr = ErrAccessDenied // no access without correct key
 	case crypto.ErrIncompatibleEncryptionMethod:
 		apiErr = ErrIncompatibleEncryptionMethod
@@ -1443,7 +1481,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case crypto.ErrKMSAuthLogin:
 		apiErr = ErrKMSAuthFailure
-	case context.Canceled, context.DeadlineExceeded:
+	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	}
 	switch err {
@@ -1527,8 +1565,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrParseUnsupportedSyntax
 	case s3select.ErrParseUnknownOperator:
 		apiErr = ErrParseUnknownOperator
-	case s3select.ErrParseInvalidPathComponent:
-		apiErr = ErrParseInvalidPathComponent
 	case s3select.ErrParseMissingIdentAfterAt:
 		apiErr = ErrParseMissingIdentAfterAt
 	case s3select.ErrParseUnexpectedOperator:
@@ -1601,8 +1637,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrEvaluatorInvalidTimestampFormatPatternToken
 	case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbol:
 		apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbol
-	case s3select.ErrInvalidColumnIndex:
-		apiErr = ErrInvalidColumnIndex
 	case s3select.ErrEvaluatorBindingDoesNotExist:
 		apiErr = ErrEvaluatorBindingDoesNotExist
 	case s3select.ErrMissingHeaders:
@@ -1610,6 +1644,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 	}
 
+	// Compression errors
+	switch err {
+	case errInvalidDecompressedSize:
+		apiErr = ErrInvalidDecompressedSize
+	}
+
 	if apiErr != ErrNone {
 		// If there was a match in the above switch case.
 		return apiErr
@@ -1721,7 +1761,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 // getAPIError provides API Error for input API error code.
 func getAPIError(code APIErrorCode) APIError {
-	return errorCodeResponse[code]
+	if apiErr, ok := errorCodeResponse[code]; ok {
+		return apiErr
+	}
+	return errorCodeResponse[ErrInternalError]
 }
 
 // getErrorResponse gets in standard error and resource value and

cmd/api-errors_test.go

@@ -52,7 +52,6 @@ var toAPIErrorCodeTests = []struct {
 	{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
 
 	// SSE-C errors
-	{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
 	{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
 	{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
 	{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},

cmd/api-headers.go

@@ -24,6 +24,8 @@ import (
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/minio/minio/cmd/crypto"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -41,6 +43,9 @@ func setCommonHeaders(w http.ResponseWriter) {
 		w.Header().Set("X-Amz-Bucket-Region", region)
 	}
 	w.Header().Set("Accept-Ranges", "bytes")
+
+	// Remove sensitive information
+	crypto.RemoveSensitiveHeaders(w.Header())
 }
 
 // Encodes the response headers into XML format.
@@ -61,13 +66,10 @@ func encodeResponseJSON(response interface{}) []byte {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
 	// set common headers
 	setCommonHeaders(w)
 
-	// Set content length.
-	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
-
 	// Set last modified time.
 	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
 	w.Header().Set("Last-Modified", lastModified)
@@ -95,10 +97,34 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set(k, v)
 	}
 
-	// for providing ranged content
-	if contentRange != nil && contentRange.offsetBegin > -1 {
-		// Override content-length
-		w.Header().Set("Content-Length", strconv.FormatInt(contentRange.getLength(), 10))
-		w.Header().Set("Content-Range", contentRange.String())
+	var totalObjectSize int64
+	switch {
+	case crypto.IsEncrypted(objInfo.UserDefined):
+		totalObjectSize, err = objInfo.DecryptedSize()
+		if err != nil {
+			return err
+		}
+	case objInfo.IsCompressed():
+		totalObjectSize = objInfo.GetActualSize()
+		if totalObjectSize < 0 {
+			return errInvalidDecompressedSize
+		}
+	default:
+		totalObjectSize = objInfo.Size
 	}
+
+	// for providing ranged content
+	start, rangeLen, err := rs.GetOffsetLength(totalObjectSize)
+	if err != nil {
+		return err
+	}
+
+	// Set content length.
+	w.Header().Set("Content-Length", strconv.FormatInt(rangeLen, 10))
+	if rs != nil {
+		contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize)
+		w.Header().Set("Content-Range", contentRange)
+	}
+
+	return nil
 }

cmd/api-resources.go

@@ -22,15 +22,22 @@ import (
 )
 
 // Parse bucket url queries
-func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
-	prefix = values.Get("prefix")
-	marker = values.Get("marker")
-	delimiter = values.Get("delimiter")
+func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
 	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
 	} else {
 		maxkeys = maxObjectList
 	}
+
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
+	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
 }
@@ -47,44 +54,69 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 		}
 	}
 
+	if values.Get("max-keys") != "" {
+		var err error
+		if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
+			errCode = ErrInvalidMaxKeys
+			return
+		}
+	} else {
+		maxkeys = maxObjectList
+	}
+
 	prefix = values.Get("prefix")
 	token = values.Get("continuation-token")
 	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-keys") != "" {
-		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
-	} else {
-		maxkeys = maxObjectList
-	}
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse bucket url queries for ?uploads
-func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
+func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string, errCode APIErrorCode) {
+	errCode = ErrNone
+
+	if values.Get("max-uploads") != "" {
+		var err error
+		if maxUploads, err = strconv.Atoi(values.Get("max-uploads")); err != nil {
+			errCode = ErrInvalidMaxUploads
+			return
+		}
+	} else {
+		maxUploads = maxUploadsList
+	}
+
 	prefix = values.Get("prefix")
 	keyMarker = values.Get("key-marker")
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
-	if values.Get("max-uploads") != "" {
-		maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
-	} else {
-		maxUploads = maxUploadsList
-	}
 	encodingType = values.Get("encoding-type")
 	return
 }
 
 // Parse object url queries
-func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
-	uploadID = values.Get("uploadId")
-	partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
+func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string, errCode APIErrorCode) {
+	var err error
+	errCode = ErrNone
+
 	if values.Get("max-parts") != "" {
-		maxParts, _ = strconv.Atoi(values.Get("max-parts"))
+		if maxParts, err = strconv.Atoi(values.Get("max-parts")); err != nil {
+			errCode = ErrInvalidMaxParts
+			return
+		}
 	} else {
 		maxParts = maxPartsList
 	}
+
+	if values.Get("part-number-marker") != "" {
+		if partNumberMarker, err = strconv.Atoi(values.Get("part-number-marker")); err != nil {
+			errCode = ErrInvalidPartNumberMarker
+			return
+		}
+	}
+
+	uploadID = values.Get("uploadId")
 	encodingType = values.Get("encoding-type")
 	return
 }

cmd/api-resources_test.go

@@ -156,7 +156,10 @@ func TestListObjectsV1Resources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		prefix, marker, delimiter, maxKeys, encodingType := getListObjectsV1Args(testCase.values)
+		prefix, marker, delimiter, maxKeys, encodingType, argsErr := getListObjectsV1Args(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if prefix != testCase.prefix {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
 		}
@@ -198,7 +201,10 @@ func TestGetObjectsResources(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		uploadID, partNumberMarker, maxParts, encodingType := getObjectResources(testCase.values)
+		uploadID, partNumberMarker, maxParts, encodingType, argsErr := getObjectResources(testCase.values)
+		if argsErr != ErrNone {
+			t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
+		}
 		if uploadID != testCase.uploadID {
 			t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID)
 		}

cmd/api-response.go

@@ -608,7 +608,7 @@ func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
 		Code:      apiError.Code,
 		Message:   errBody,
 		Resource:  reqURL.Path,
-		RequestID: "3L137",
+		RequestID: w.Header().Get(responseRequestIDKey),
 		HostID:    "3L137",
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)

cmd/auth-handler.go

@@ -27,8 +27,10 @@ import (
 	"net/http"
 	"strings"
 
+	jwtgo "github.com/dgrijalva/jwt-go"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -86,6 +88,7 @@ const (
 	authTypeSigned
 	authTypeSignedV2
 	authTypeJWT
+	authTypeSTS
 )
 
 // Get request authentication type.
@@ -106,6 +109,8 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypePostPolicy
 	} else if _, ok := r.Header["Authorization"]; !ok {
 		return authTypeAnonymous
+	} else if _, ok := r.URL.Query()["Action"]; ok {
+		return authTypeSTS
 	}
 	return authTypeUnknown
 }
@@ -114,7 +119,21 @@ func getRequestAuthType(r *http.Request) authType {
 // It does not accept presigned or JWT or anonymous requests.
 func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	s3Err := ErrAccessDenied
-	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok && getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) { // we only support V4 (no presign) with auth. body
+	if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
+		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		// We only support admin credentials to access admin APIs.
+
+		var owner bool
+		_, owner, s3Err = getReqAccessKeyV4(r, region)
+		if s3Err != ErrNone {
+			return s3Err
+		}
+
+		if !owner {
+			return ErrAccessDenied
+		}
+
+		// we only support V4 (no presign) with auth body
 		s3Err = isReqAuthenticated(r, region)
 	}
 	if s3Err != ErrNone {
@@ -125,30 +144,62 @@ func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
 	return s3Err
 }
 
-func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
-	isOwner := true
-	accountName := globalServerConfig.GetCredential().AccessKey
+// Fetch the security token set by the client.
+func getSessionToken(r *http.Request) (token string) {
+	token = r.Header.Get("X-Amz-Security-Token")
+	if token != "" {
+		return token
+	}
+	return r.URL.Query().Get("X-Amz-Security-Token")
+}
 
+// Fetch claims in the security token returned by the client and validate the token.
+func getClaimsFromToken(r *http.Request) (map[string]interface{}, APIErrorCode) {
+	claims := make(map[string]interface{})
+	token := getSessionToken(r)
+	if token == "" {
+		return nil, ErrNone
+	}
+	p := &jwtgo.Parser{}
+	jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
+	if err != nil {
+		return nil, toAPIErrorCode(errAuthentication)
+	}
+	if !jtoken.Valid {
+		return nil, toAPIErrorCode(errAuthentication)
+	}
+	return claims, ErrNone
+}
+
+// Check request auth type verifies the incoming http request
+// - validates the request signature
+// - validates the policy action if anonymous tests bucket policies if any,
+//   for authenticated requests validates IAM policies.
+// returns APIErrorCode if any to be replied to the client.
+func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
+	var accessKey string
+	var owner bool
 	switch getRequestAuthType(r) {
-	case authTypeUnknown:
+	case authTypeUnknown, authTypeStreamingSigned:
 		return ErrAccessDenied
 	case authTypePresignedV2, authTypeSignedV2:
-		if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
+			return s3Err
 		}
+		accessKey, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
 		region := globalServerConfig.GetRegion()
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
 		}
-
-		if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
-			return errorCode
+		if s3Err = isReqAuthenticated(r, region); s3Err != ErrNone {
+			return s3Err
 		}
-	default:
-		isOwner = false
-		accountName = ""
+		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
 	}
 
 	// LocationConstraint is valid only for CreateBucketAction.
@@ -174,17 +225,36 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
 
-	if globalPolicySys.IsAllowed(policy.Args{
-		AccountName:     accountName,
-		Action:          action,
+	claims, s3Err := getClaimsFromToken(r)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if accessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     accessKey,
+			Action:          action,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, locationConstraint),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     accessKey,
+		Action:          iampolicy.Action(action),
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, locationConstraint),
-		IsOwner:         isOwner,
+		ConditionValues: getConditionValues(r, ""),
 		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
 	}) {
 		return ErrNone
 	}
-
 	return ErrAccessDenied
 }
 
@@ -244,7 +314,7 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256))
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
 		return toAPIErrorCode(err)
 	}
@@ -288,7 +358,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	} else if aType == authTypeJWT {
 		// Validate Authorization header if its valid for JWT request.
-		if !isHTTPRequestValid(r) {
+		if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 			w.WriteHeader(http.StatusUnauthorized)
 			return
 		}
@@ -297,3 +367,55 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 	writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
 }
+
+// isPutAllowed - check if PUT operation is allowed on the resource, this
+// call verifies bucket policies and IAM policies, supports multi user
+// checks etc.
+func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
+	var accessKey string
+	var owner bool
+	switch atype {
+	case authTypeUnknown:
+		return ErrAccessDenied
+	case authTypeSignedV2, authTypePresignedV2:
+		accessKey, owner, s3Err = getReqAccessKeyV2(r)
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
+		region := globalServerConfig.GetRegion()
+		accessKey, owner, s3Err = getReqAccessKeyV4(r, region)
+	}
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	claims, s3Err := getClaimsFromToken(r)
+	if s3Err != ErrNone {
+		return s3Err
+	}
+
+	if accessKey == "" {
+		if globalPolicySys.IsAllowed(policy.Args{
+			AccountName:     accessKey,
+			Action:          policy.PutObjectAction,
+			BucketName:      bucketName,
+			ConditionValues: getConditionValues(r, ""),
+			IsOwner:         false,
+			ObjectName:      objectName,
+		}) {
+			return ErrNone
+		}
+		return ErrAccessDenied
+	}
+
+	if globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     accessKey,
+		Action:          policy.PutObjectAction,
+		BucketName:      bucketName,
+		ConditionValues: getConditionValues(r, ""),
+		ObjectName:      objectName,
+		IsOwner:         owner,
+		Claims:          claims,
+	}) {
+		return ErrNone
+	}
+	return ErrAccessDenied
+}

cmd/benchmark-utils_test.go

@@ -61,7 +61,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -97,7 +97,8 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata)
+	opts := ObjectOptions{}
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata, opts)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -122,7 +123,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			md5hex = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
+				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -203,7 +204,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -218,7 +219,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		var buffer = new(bytes.Buffer)
-		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "")
+		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "", ObjectOptions{})
 		if err != nil {
 			b.Error(err)
 		}
@@ -303,7 +304,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -343,7 +344,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -359,7 +360,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	b.RunParallel(func(pb *testing.PB) {
 		i := 0
 		for pb.Next() {
-			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "")
+			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "", ObjectOptions{})
 			if err != nil {
 				b.Error(err)
 			}

cmd/bitrot.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/minio/highwayhash"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/sha256-simd"
+	sha256 "github.com/minio/sha256-simd"
 	"golang.org/x/crypto/blake2b"
 )
 
@@ -70,7 +70,7 @@ func (a BitrotAlgorithm) New() hash.Hash {
 	}
 }
 
-// Available reports whether the given algorihm is available.
+// Available reports whether the given algorithm is available.
 func (a BitrotAlgorithm) Available() bool {
 	_, ok := bitrotAlgorithms[a]
 	return ok
@@ -138,7 +138,9 @@ func (b *bitrotReader) ReadChunk(offset int64, length int64) ([]byte, error) {
 	if b.buf == nil {
 		b.buf = make([]byte, b.endOffset-offset)
 		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(context.Background(), err)
+			ctx := context.Background()
+			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 	}

cmd/bucket-handlers-listobjects.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/policy"
 )
@@ -58,6 +59,8 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -102,7 +105,17 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+		var actualSize int64
+		if listObjectsV2Info.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsV2Info.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -127,6 +140,8 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV1")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -142,7 +157,11 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	// Extract all the litsObjectsV1 query params to their native values.
-	prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
+	prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 
 	// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
 	// does not throw an error.
@@ -168,7 +187,17 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsInfo.Objects {
-		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+		var actualSize int64
+		if listObjectsInfo.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsInfo.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsInfo.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -176,7 +205,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 			}
 		}
 	}
-
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
 
 	// Write success response.

cmd/bucket-handlers.go

@@ -89,6 +89,8 @@ func initFederatorBackend(objLayer ObjectLayer) {
 func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketLocation")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -137,6 +139,8 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListMultipartUploads")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -151,7 +155,11 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 		return
 	}
 
-	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
+	prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
+	if s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL)
+		return
+	}
 	if maxUploads < 0 {
 		writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
 		return
@@ -184,6 +192,8 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBuckets")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -240,6 +250,8 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteMultipleObjects")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -370,10 +382,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			Object: ObjectInfo{
 				Name: dobj.ObjectName,
 			},
-			ReqParams: extractReqParams(r),
-			UserAgent: r.UserAgent(),
-			Host:      host,
-			Port:      port,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         host,
+			Port:         port,
 		})
 	}
 }
@@ -384,6 +397,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -460,6 +475,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PostPolicyBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -590,7 +607,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -614,7 +631,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "") // do not try to verify encrypted content
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return
@@ -622,7 +639,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata)
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata, ObjectOptions{})
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -640,13 +657,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Notify object created event.
 	defer sendEvent(eventArgs{
-		EventName:  event.ObjectCreatedPost,
-		BucketName: objInfo.Bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectCreatedPost,
+		BucketName:   objInfo.Bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 
 	if successRedirect != "" {
@@ -682,6 +700,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -712,6 +732,8 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucket")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 

cmd/bucket-handlers_test.go

@@ -84,7 +84,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get bucket location.
-		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -116,7 +116,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -220,7 +220,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for HEAD bucket.
-		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -235,7 +235,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -437,7 +437,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// construct HTTP request for List multipart uploads endpoint.
 		u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
-		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if gerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
 		}
@@ -454,7 +454,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -471,7 +471,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 	// construct HTTP request for List multipart uploads endpoint.
 	u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
-	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "") // Generate an anonymous request.
+	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "", nil) // Generate an anonymous request.
 	if err != nil {
 		t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
 	}
@@ -551,7 +551,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if lerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
 		}
@@ -568,7 +568,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
+		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)
@@ -745,7 +745,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		// Generate a signed or anonymous request based on the testCase
 		if testCase.accessKey != "" {
 			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
-				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil)
 		} else {
 			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
@@ -785,7 +785,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	nilBucket := "dummy-bucket"
 	nilObject := ""
 
-	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "")
+	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "", nil)
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}

cmd/bucket-notification-handlers.go

@@ -44,6 +44,8 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
 func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketNotification")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 
@@ -96,6 +98,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketNotification")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -139,7 +143,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if err = saveNotificationConfig(objectAPI, bucketName, config); err != nil {
+	if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
@@ -156,6 +160,8 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListenBucketNotification")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	// Validate if bucket exists.
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {

cmd/bucket-policy-handlers.go

@@ -40,6 +40,8 @@ const (
 func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutBucketPolicy")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -101,6 +103,8 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DeleteBucketPolicy")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
@@ -137,6 +141,8 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetBucketPolicy")
 
+	defer logger.AuditLog(ctx, w, r)
+
 	objAPI := api.ObjectAPI()
 	if objAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)

cmd/bucket-policy-handlers_test.go

@@ -254,7 +254,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -268,7 +268,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -304,7 +304,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -346,7 +346,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -360,7 +360,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -417,7 +417,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -456,7 +456,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -511,7 +511,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("GET", getGetPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -591,7 +591,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -641,7 +641,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -663,7 +663,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -680,7 +680,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -714,7 +714,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)

cmd/certs.go

@@ -150,24 +150,20 @@ func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
 	return cert, nil
 }
 
-func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
+func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
-		return nil, nil, nil, false, nil
+		return nil, nil, false, nil
 	}
 
 	if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
 	if err != nil {
-		return nil, nil, nil, false, err
-	}
-
-	if rootCAs, err = getRootCAs(getCADir()); err != nil {
-		return nil, nil, nil, false, err
+		return nil, nil, false, err
 	}
 
 	secureConn = true
-	return x509Certs, rootCAs, c, secureConn, nil
+	return x509Certs, c, secureConn, nil
 }

cmd/common-main.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"crypto/tls"
 	"errors"
 	"net"
 	"os"
@@ -26,13 +27,14 @@ import (
 	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
+	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
+	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
-
-	"github.com/minio/minio-go/pkg/set"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Check for updates and print a notification message
@@ -49,16 +51,30 @@ func checkUpdate(mode string) {
 
 // Load logger targets based on user's configuration
 func loadLoggers() {
+	auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable audit HTTP logging through ENV.
+		logger.AddAuditTarget(logger.NewHTTP(auditEndpoint, NewCustomHTTPTransport()))
+	}
+
+	loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
+	if ok {
+		// Enable HTTP logging through ENV.
+		logger.AddTarget(logger.NewHTTP(loggerEndpoint, NewCustomHTTPTransport()))
+	} else {
+		for _, l := range globalServerConfig.Logger.HTTP {
+			if l.Enabled {
+				// Enable http logging
+				logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
+			}
+		}
+	}
+
 	if globalServerConfig.Logger.Console.Enabled {
 		// Enable console logging
 		logger.AddTarget(logger.NewConsole())
 	}
-	for _, l := range globalServerConfig.Logger.HTTP {
-		if l.Enabled {
-			// Enable http logging
-			logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
-		}
-	}
+
 }
 
 func handleCommonCmdArgs(ctx *cli.Context) {
@@ -95,10 +111,23 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	setConfigDir(configDirAbs)
 }
 
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(includes []string) ([]string, error) {
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+		}
+	}
+	return includes, nil
+}
+
 func handleCommonEnvVars() {
+	compressEnvDelimiter := ","
 	// Start profiler if env is set.
 	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		globalProfiler = startProfiler(profiler)
+		var err error
+		globalProfiler, err = startProfiler(profiler, "")
+		logger.FatalIf(err, "Unable to setup a profiler")
 	}
 
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")
@@ -108,21 +137,17 @@ func handleCommonEnvVars() {
 		if err != nil {
 			logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
 		}
+		cred.Expiration = timeSentinel
 
 		// credential Envs are set globally.
 		globalIsEnvCreds = true
 		globalActiveCred = cred
 	}
 
-	// In distributed setup users need to set ENVs always.
-	if !globalIsEnvCreds && globalIsDistXL {
-		logger.Fatal(uiErrEnvCredentialsMissingServer(nil), "Unable to start distributed server mode")
-	}
-
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
 		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Unable to validate MINIO_BROWSER environment variable")
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -141,16 +166,55 @@ func handleCommonEnvVars() {
 	etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
 	if ok {
 		etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
+
+		var etcdSecure bool
+		for _, endpoint := range etcdEndpoints {
+			u, err := xnet.ParseURL(endpoint)
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
+			}
+			// If one of the endpoint is https, we will use https directly.
+			etcdSecure = etcdSecure || u.Scheme == "https"
+		}
+
 		var err error
-		globalEtcdClient, err = etcd.New(etcd.Config{
-			Endpoints:         etcdEndpoints,
-			DialTimeout:       defaultDialTimeout,
-			DialKeepAliveTime: defaultDialKeepAlive,
-		})
+		if etcdSecure {
+			// This is only to support client side certificate authentication
+			// https://coreos.com/etcd/docs/latest/op-guide/security.html
+			etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
+			etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
+			var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
+			if ok1 && ok2 {
+				getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+					cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
+					return &cert, terr
+				}
+			}
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+				TLS: &tls.Config{
+					RootCAs:              globalRootCAs,
+					GetClientCertificate: getClientCertificate,
+				},
+			})
+		} else {
+			globalEtcdClient, err = etcd.New(etcd.Config{
+				Endpoints:         etcdEndpoints,
+				DialTimeout:       defaultDialTimeout,
+				DialKeepAliveTime: defaultDialKeepAlive,
+			})
+		}
 		logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
 	}
 
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
+	if globalDomainName != "" {
+		if _, ok = dns2.IsDomainName(globalDomainName); !ok {
+			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN value in environment variable")
+		}
+	}
 
 	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
 	if ok {
@@ -243,7 +307,7 @@ func handleCommonEnvVars() {
 	if worm := os.Getenv("MINIO_WORM"); worm != "" {
 		wormFlag, err := ParseBoolFlag(worm)
 		if err != nil {
-			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Unable to validate MINIO_WORM environment variable")
+			logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
 		}
 
 		// worm Envs are set globally, this does not represent
@@ -265,4 +329,28 @@ func handleCommonEnvVars() {
 		globalKMSKeyID = kmsConf.Vault.Key.Name
 		globalKMSConfig = kmsConf
 	}
+
+	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
+		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
+	}
+
+	compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS")
+	compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES")
+	if compressExtensions != "" || compressMimeTypes != "" {
+		globalIsEnvCompression = true
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions)
+			}
+			globalCompressExtensions = extensions
+		}
+		if compressMimeTypes != "" {
+			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes)
+			}
+			globalCompressMimeTypes = contenttypes
+		}
+	}
 }

cmd/config-common.go

@@ -0,0 +1,133 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+
+	etcd "github.com/coreos/etcd/clientv3"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+)
+
+var errConfigNotFound = errors.New("config file not found")
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	var buffer bytes.Buffer
+	// Read entire content by setting size to -1
+	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return nil, errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return nil, err
+	}
+
+	// Return config not found on empty content.
+	if buffer.Len() == 0 {
+		return nil, errConfigNotFound
+	}
+
+	return buffer.Bytes(), nil
+}
+
+func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	_, err := client.Delete(ctx, configFile)
+	return err
+}
+
+func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
+}
+
+func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
+	_, err := client.Put(ctx, configFile, string(data))
+	return err
+}
+
+func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	if err != nil {
+		return err
+	}
+
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
+	return err
+}
+
+func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
+	resp, err := client.Get(ctx, configFile)
+	if err != nil {
+		return nil, err
+	}
+	if resp.Count == 0 {
+		return nil, errConfigNotFound
+	}
+	for _, ev := range resp.Kvs {
+		if string(ev.Key) == configFile {
+			return ev.Value, nil
+		}
+	}
+	return nil, errConfigNotFound
+}
+
+// watchConfig - watches for changes on `configFile` on etcd and loads them.
+func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+	if globalEtcdClient != nil {
+		for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
+			for _, event := range watchResp.Events {
+				if event.IsModify() || event.IsCreate() {
+					loadCfgFn(objAPI)
+				}
+			}
+		}
+	}
+}
+
+func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
+	resp, err := globalEtcdClient.Get(ctx, configFile)
+	if err != nil {
+		return err
+	}
+	if resp.Count == 0 {
+		return errConfigNotFound
+	}
+	return nil
+}
+
+func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
+	if globalEtcdClient != nil {
+		return checkConfigEtcd(ctx, globalEtcdClient, configFile)
+	}
+
+	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
+		// Treat object not found as config not found.
+		if isErrObjectNotFound(err) {
+			return errConfigNotFound
+		}
+
+		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
+		logger.LogIf(ctx, err)
+		return err
+	}
+	return nil
+}

cmd/config-current.go

@@ -20,16 +20,18 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"os"
 	"reflect"
 	"sync"
 
-	"github.com/miekg/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
-
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Steps to move from version N to version N+1
@@ -41,9 +43,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "28"
+const serverConfigVersion = "31"
 
-type serverConfig = serverConfigV28
+type serverConfig = serverConfigV31
 
 var (
 	// globalServerConfig server config.
@@ -97,12 +99,6 @@ func (s *serverConfig) GetCredential() auth.Credentials {
 	return s.Credential
 }
 
-// SetBrowser set if browser is enabled.
-func (s *serverConfig) SetBrowser(b bool) {
-	// Set the new value.
-	s.Browser = BoolFlag(b)
-}
-
 // SetWorm set if worm is enabled.
 func (s *serverConfig) SetWorm(b bool) {
 	// Set the new value.
@@ -126,21 +122,10 @@ func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
 	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
-// GetBrowser get current credentials.
-func (s *serverConfig) GetBrowser() bool {
-	if globalIsEnvWORM {
-		return globalWORMEnabled
-	}
-	if s == nil {
-		return true
-	}
-	return bool(s.Browser)
-}
-
 // GetWorm get current credentials.
 func (s *serverConfig) GetWorm() bool {
-	if globalIsEnvBrowser {
-		return globalIsBrowserEnabled
+	if globalIsEnvWORM {
+		return globalWORMEnabled
 	}
 	if s == nil {
 		return false
@@ -188,81 +173,82 @@ func (s *serverConfig) Validate() error {
 	}
 
 	// Region: nothing to validate
-	// Browser, Worm, Cache and StorageClass values are already validated during json unmarshal
-
-	if s.Domain != "" {
-		if _, ok := dns.IsDomainName(s.Domain); !ok {
-			return errors.New("invalid domain name")
-		}
-	}
-
+	// Worm, Cache and StorageClass values are already validated during json unmarshal
 	for _, v := range s.Notify.AMQP {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("amqp: %s", err.Error())
+			return fmt.Errorf("amqp: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Elasticsearch {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("elasticsearch: %s", err.Error())
+			return fmt.Errorf("elasticsearch: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Kafka {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("kafka: %s", err.Error())
+			return fmt.Errorf("kafka: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.MQTT {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("mqtt: %s", err.Error())
+			return fmt.Errorf("mqtt: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.MySQL {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("mysql: %s", err.Error())
+			return fmt.Errorf("mysql: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.NATS {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("nats: %s", err.Error())
+			return fmt.Errorf("nats: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.PostgreSQL {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("postgreSQL: %s", err.Error())
+			return fmt.Errorf("postgreSQL: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Redis {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("redis: %s", err.Error())
+			return fmt.Errorf("redis: %s", err)
 		}
 	}
 
 	for _, v := range s.Notify.Webhook {
 		if err := v.Validate(); err != nil {
-			return fmt.Errorf("webhook: %s", err.Error())
+			return fmt.Errorf("webhook: %s", err)
 		}
 	}
 
 	return nil
 }
 
+// SetCompressionConfig sets the current compression config
+func (s *serverConfig) SetCompressionConfig(extensions []string, mimeTypes []string) {
+	s.Compression.Extensions = extensions
+	s.Compression.MimeTypes = mimeTypes
+	s.Compression.Enabled = globalIsCompressionEnabled
+}
+
+// GetCompressionConfig gets the current compression config
+func (s *serverConfig) GetCompressionConfig() compressionConfig {
+	return s.Compression
+}
+
 func (s *serverConfig) loadFromEnvs() {
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
 		s.SetCredential(globalActiveCred)
 	}
 
-	if globalIsEnvBrowser {
-		s.SetBrowser(globalIsBrowserEnabled)
-	}
-
 	if globalIsEnvWORM {
 		s.SetWorm(globalWORMEnabled)
 	}
@@ -271,10 +257,6 @@ func (s *serverConfig) loadFromEnvs() {
 		s.SetRegion(globalServerRegion)
 	}
 
-	if globalIsEnvDomainName {
-		s.Domain = globalDomainName
-	}
-
 	if globalIsStorageClass {
 		s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
 	}
@@ -286,6 +268,120 @@ func (s *serverConfig) loadFromEnvs() {
 	if globalKMS != nil {
 		s.KMS = globalKMSConfig
 	}
+
+	if globalIsEnvCompression {
+		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
+	}
+
+	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
+		if u, err := xnet.ParseURL(jwksURL); err == nil {
+			s.OpenID.JWKS.URL = u
+			s.OpenID.JWKS.PopulatePublicKey()
+		}
+	}
+
+	if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
+		if u, err := xnet.ParseURL(opaURL); err == nil {
+			s.Policy.OPA.URL = u
+			s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
+		}
+	}
+}
+
+// TestNotificationTargets tries to establish connections to all notification
+// targets when enabled. This is a good way to make sure all configurations
+// set by the user can work.
+func (s *serverConfig) TestNotificationTargets() error {
+	for k, v := range s.Notify.AMQP {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewAMQPTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("amqp(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Elasticsearch {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewElasticsearchTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Kafka {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewKafkaTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("kafka(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MQTT {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMQTTTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mqtt(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MySQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMySQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mysql(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.NATS {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewNATSTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("nats(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.PostgreSQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewPostgreSQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Redis {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewRedisTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("redis(%s): %s", k, err.Error())
+		}
+		t.Close()
+
+	}
+
+	return nil
 }
 
 // Returns the string describing a difference with the given
@@ -299,14 +395,12 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "Credential configuration differs"
 	case s.Region != t.Region:
 		return "Region configuration differs"
-	case s.Browser != t.Browser:
-		return "Browser configuration differs"
-	case s.Domain != t.Domain:
-		return "Domain configuration differs"
 	case s.StorageClass != t.StorageClass:
 		return "StorageClass configuration differs"
 	case !reflect.DeepEqual(s.Cache, t.Cache):
 		return "Cache configuration differs"
+	case !reflect.DeepEqual(s.Compression, t.Compression):
+		return "Compression configuration differs"
 	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
@@ -346,7 +440,6 @@ func newServerConfig() *serverConfig {
 		Version:    serverConfigVersion,
 		Credential: cred,
 		Region:     globalMinioDefaultRegion,
-		Browser:    true,
 		StorageClass: storageClassConfig{
 			Standard: storageClass{},
 			RRS:      storageClass{},
@@ -359,6 +452,11 @@ func newServerConfig() *serverConfig {
 		},
 		KMS:    crypto.KMSConfig{},
 		Notify: notifier{},
+		Compression: compressionConfig{
+			Enabled:    false,
+			Extensions: globalCompressExtensions,
+			MimeTypes:  globalCompressMimeTypes,
+		},
 	}
 
 	// Make sure to initialize notification configs.
@@ -399,18 +497,12 @@ func (s *serverConfig) loadToCachedConfigs() {
 	if !globalIsEnvCreds {
 		globalActiveCred = s.GetCredential()
 	}
-	if !globalIsEnvBrowser {
-		globalIsBrowserEnabled = s.GetBrowser()
-	}
 	if !globalIsEnvWORM {
 		globalWORMEnabled = s.GetWorm()
 	}
 	if !globalIsEnvRegion {
 		globalServerRegion = s.GetRegion()
 	}
-	if !globalIsEnvDomainName {
-		globalDomainName = s.Domain
-	}
 	if !globalIsStorageClass {
 		globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
 	}
@@ -428,11 +520,31 @@ func (s *serverConfig) loadToCachedConfigs() {
 			globalKMSKeyID = globalKMSConfig.Vault.Key.Name
 		}
 	}
+
+	if !globalIsCompressionEnabled {
+		compressionConf := s.GetCompressionConfig()
+		globalCompressExtensions = compressionConf.Extensions
+		globalCompressMimeTypes = compressionConf.MimeTypes
+		globalIsCompressionEnabled = compressionConf.Enabled
+	}
+
+	if globalIAMValidators == nil {
+		globalIAMValidators = getAuthValidators(s)
+	}
+
+	if globalPolicyOPA == nil {
+		if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
+			globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+				URL:       s.Policy.OPA.URL,
+				AuthToken: s.Policy.OPA.AuthToken,
+			})
+		}
+	}
 }
 
-// newConfig - initialize a new server config, saves env parameters if
+// newSrvConfig - initialize a new server config, saves env parameters if
 // found, otherwise use default parameters
-func newConfig(objAPI ObjectLayer) error {
+func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
@@ -448,7 +560,7 @@ func newConfig(objAPI ObjectLayer) error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return saveServerConfig(objAPI, globalServerConfig)
+	return saveServerConfig(context.Background(), objAPI, globalServerConfig)
 }
 
 // getValidConfig - returns valid server configuration
@@ -483,6 +595,20 @@ func loadConfig(objAPI ObjectLayer) error {
 	return nil
 }
 
+// getAuthValidators - returns ValidatorList which contains
+// enabled providers in server config.
+// A new authentication provider is added like below
+// * Add a new provider in pkg/iam/validator package.
+func getAuthValidators(config *serverConfig) *validator.Validators {
+	validators := validator.NewValidators()
+
+	if config.OpenID.JWKS.URL != nil {
+		validators.Add(validator.NewJWT(config.OpenID.JWKS))
+	}
+
+	return validators
+}
+
 // getNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
 // A new notification target is added like below
 // * Add a new target in pkg/event/target package.

cmd/config-current_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"os"
 	"path"
 	"testing"
@@ -51,7 +52,7 @@ func TestServerConfig(t *testing.T) {
 		t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
 	}
 
-	if err := saveServerConfig(objLayer, globalServerConfig); err != nil {
+	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
@@ -66,6 +67,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_BROWSER", "off")
 	defer os.Unsetenv("MINIO_BROWSER")
 
+	os.Setenv("MINIO_WORM", "on")
+	defer os.Unsetenv("MINIO_WORM")
+
 	os.Setenv("MINIO_ACCESS_KEY", "minio")
 	defer os.Unsetenv("MINIO_ACCESS_KEY")
 
@@ -99,29 +103,35 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	// Init config
 	initConfig(objLayer)
 
-	// Check if serverConfig has
-	if globalServerConfig.GetBrowser() {
-		t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
+	// Check if serverConfig has browser disabled
+	if globalIsBrowserEnabled {
+		t.Error("Expected browser to be disabled but it is not")
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig returns WORM config from the env
+	if !globalServerConfig.GetWorm() {
+		t.Error("Expected WORM to be enabled but it is not")
+	}
+
+	// Check if serverConfig has region from the environment
 	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region to be \"us-west-1\" found %v", globalServerConfig.GetRegion())
+		t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig has credentials from the environment
 	cred := globalServerConfig.GetCredential()
 
 	if cred.AccessKey != "minio" {
-		t.Errorf("Expecting access key to be `minio` found %s", cred.AccessKey)
+		t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
 	}
 
 	if cred.SecretKey != "minio123" {
-		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
+		t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
 	}
 
-	if globalServerConfig.Domain != "domain.com" {
-		t.Errorf("Expecting Domain to be `domain.com` found " + globalServerConfig.Domain)
+	// Check if serverConfig has the correct domain
+	if globalDomainName != "domain.com" {
+		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainName)
 	}
 }
 
@@ -227,7 +237,7 @@ func TestValidateConfig(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		if err = saveConfig(objLayer, configPath, []byte(testCase.configData)); err != nil {
+		if err = saveConfig(context.Background(), objLayer, configPath, []byte(testCase.configData)); err != nil {
 			t.Fatal(err)
 		}
 		_, err = getValidConfig(objLayer)
@@ -250,77 +260,81 @@ func TestConfigDiff(t *testing.T) {
 		{&serverConfig{}, nil, "Given configuration is empty"},
 		// 2
 		{
-			&serverConfig{Credential: auth.Credentials{"u1", "p1"}},
-			&serverConfig{Credential: auth.Credentials{"u1", "p2"}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p1",
+				Expiration: timeSentinel,
+			}},
+			&serverConfig{Credential: auth.Credentials{
+				AccessKey:  "u1",
+				SecretKey:  "p2",
+				Expiration: timeSentinel,
+			}},
 			"Credential configuration differs",
 		},
 		// 3
 		{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
 		// 4
-		{&serverConfig{Browser: false}, &serverConfig{Browser: true}, "Browser configuration differs"},
-		// 5
-		{&serverConfig{Domain: "domain1"}, &serverConfig{Domain: "domain2"}, "Domain configuration differs"},
-		// 6
 		{
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
 			"StorageClass configuration differs",
 		},
-		// 7
+		// 5
 		{
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
 			"AMQP Notification configuration differs",
 		},
-		// 8
+		// 6
 		{
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
 			"NATS Notification configuration differs",
 		},
-		// 9
+		// 7
 		{
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
 			"ElasticSearch Notification configuration differs",
 		},
-		// 10
+		// 8
 		{
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
 			"Redis Notification configuration differs",
 		},
-		// 11
+		// 9
 		{
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
 			"PostgreSQL Notification configuration differs",
 		},
-		// 12
+		// 10
 		{
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
 			"Kafka Notification configuration differs",
 		},
-		// 13
+		// 11
 		{
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
 			"Webhook Notification configuration differs",
 		},
-		// 14
+		// 12
 		{
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
 			"MySQL Notification configuration differs",
 		},
-		// 15
+		// 13
 		{
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
-		// 16
+		// 14
 		{
 			&serverConfig{Logger: loggerConfig{
 				Console: loggerConsole{Enabled: true},

cmd/config-migrate.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"os"
 	"path"
@@ -29,6 +30,8 @@ import (
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -2387,6 +2390,7 @@ func migrateV27ToV28() error {
 
 	// config V28 is backward compatible with V27, load the old
 	// config file in serverConfigV28 struct and initialize KMSConfig
+
 	srvConfig := &serverConfigV28{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
 	if os.IsNotExist(err) {
@@ -2409,29 +2413,175 @@ func migrateV27ToV28() error {
 	return nil
 }
 
-// Migrates '.minio.sys/config.json' v27 to v28.
+// Migrates '.minio.sys/config.json' to v31.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
-	return migrateV27ToV28MinioSys(objAPI)
+	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
+		return err
+	}
+	if err := migrateV28ToV29MinioSys(objAPI); err != nil {
+		return err
+	}
+	if err := migrateV29ToV30MinioSys(objAPI); err != nil {
+		return err
+	}
+	return migrateV30ToV31MinioSys(objAPI)
+}
+
+func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
+	data, err := readConfig(context.Background(), objAPI, configFile)
+	if err != nil {
+		return false, nil, err
+	}
+
+	var versionConfig struct {
+		Version string `json:"version"`
+	}
+
+	vcfg := &versionConfig
+	if err = json.Unmarshal(data, vcfg); err != nil {
+		return false, nil, err
+	}
+	return vcfg.Version == version, data, nil
 }
 
 func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	srvConfig, err := readServerConfig(context.Background(), objAPI)
+	ok, data, err := checkConfigVersion(objAPI, configFile, "27")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %v", err)
 	}
-	if srvConfig.Version != "27" {
+	if !ok {
 		return nil
 	}
 
-	srvConfig.Version = "28"
-	srvConfig.KMS = crypto.KMSConfig{}
-	if err = saveServerConfig(objAPI, srvConfig); err != nil {
+	cfg := &serverConfigV28{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "28"
+	cfg.KMS = crypto.KMSConfig{}
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
 	return nil
 }
+
+func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "28")
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if !ok {
+		return nil
+	}
+
+	cfg := &serverConfigV29{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "29"
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
+	return nil
+}
+
+func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "29")
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if !ok {
+		return nil
+	}
+
+	cfg := &serverConfigV30{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "30"
+	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
+	cfg.Compression.Enabled = false
+	cfg.Compression.Extensions = globalCompressExtensions
+	cfg.Compression.MimeTypes = globalCompressMimeTypes
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
+	return nil
+}
+
+func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	ok, data, err := checkConfigVersion(objAPI, configFile, "30")
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if !ok {
+		return nil
+	}
+
+	cfg := &serverConfigV31{}
+	if err = json.Unmarshal(data, cfg); err != nil {
+		return err
+	}
+
+	cfg.Version = "31"
+	cfg.OpenID.JWKS = validator.JWKSArgs{
+		URL: &xnet.URL{},
+	}
+	cfg.Policy.OPA = iampolicy.OpaArgs{
+		URL:       &xnet.URL{},
+		AuthToken: "",
+	}
+
+	data, err = json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+
+	if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
+	return nil
+}

cmd/config-migrate_test.go

@@ -159,8 +159,8 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	}
 }
 
-// Test if a config migration from v2 to v28 is successfully done
-func TestServerConfigMigrateV2toV28(t *testing.T) {
+// Test if a config migration from v2 to v30 is successfully done
+func TestServerConfigMigrateV2toV30(t *testing.T) {
 	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
 		t.Fatal(err)
@@ -174,10 +174,6 @@ func TestServerConfigMigrateV2toV28(t *testing.T) {
 	}
 	defer os.RemoveAll(fsDir)
 
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
 	configPath := rootPath + "/" + minioConfigFile
 
 	// Create a corrupted config file
@@ -203,12 +199,16 @@ func TestServerConfigMigrateV2toV28(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	if err := migrateConfigToMinioSys(); err != nil {
+	if err := migrateConfigToMinioSys(objLayer); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
+	if err := migrateMinioSysConfig(objLayer); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(newObjectLayerFn()); err != nil {
+	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 
@@ -222,6 +222,7 @@ func TestServerConfigMigrateV2toV28(t *testing.T) {
 	if globalServerConfig.Credential.AccessKey != accessKey {
 		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
 	}
+
 	if globalServerConfig.Credential.SecretKey != secretKey {
 		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
 	}
@@ -318,7 +319,6 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV26ToV27(); err == nil {
 		t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
 	}
-
 	if err := migrateV27ToV28(); err == nil {
 		t.Fatal("migrateConfigV27ToV28() should fail with a corrupted json")
 	}

cmd/config-versions.go

@@ -22,6 +22,8 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
+	"github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -737,9 +739,7 @@ type serverConfigV28 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
 	Worm       BoolFlag         `json:"worm"`
-	Domain     string           `json:"domain"`
 
 	// Storage class configuration
 	StorageClass storageClassConfig `json:"storageclass"`
@@ -756,3 +756,84 @@ type serverConfigV28 struct {
 	// Logger configuration
 	Logger loggerConfig `json:"logger"`
 }
+
+// serverConfigV29 is just like version '28'.
+type serverConfigV29 serverConfigV28
+
+// compressionConfig represents the compression settings.
+type compressionConfig struct {
+	Enabled    bool     `json:"enabled"`
+	Extensions []string `json:"extensions"`
+	MimeTypes  []string `json:"mime-types"`
+}
+
+// serverConfigV30 is just like version '29', stores additionally
+// extensions and mimetypes fields for compression.
+type serverConfigV30 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+}
+
+// serverConfigV31 is just like version '30', with OPA and OpenID configuration.
+type serverConfigV31 struct {
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+
+	// OpenID configuration
+	OpenID struct {
+		// JWKS validator config.
+		JWKS validator.JWKSArgs `json:"jwks"`
+	} `json:"openid"`
+
+	// External policy enforcements.
+	Policy struct {
+		// OPA configuration.
+		OPA iampolicy.OpaArgs `json:"opa"`
+
+		// Add new external policy enforcements here.
+	} `json:"policy"`
+}

cmd/config.go

@@ -20,15 +20,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"errors"
-	"io"
-	"io/ioutil"
+	"os"
 	"path"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/quick"
 )
 
@@ -37,60 +35,55 @@ const (
 
 	// Minio configuration file.
 	minioConfigFile = "config.json"
+
+	// Minio backup file
+	minioConfigBackupFile = minioConfigFile + ".backup"
 )
 
-func saveServerConfig(objAPI ObjectLayer, config *serverConfig) error {
+func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverConfig) error {
 	if err := quick.CheckData(config); err != nil {
 		return err
 	}
 
-	data, err := json.Marshal(config)
+	data, err := json.MarshalIndent(config, "", "\t")
 	if err != nil {
 		return err
 	}
 
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	if globalEtcdClient != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-		_, err := globalEtcdClient.Put(ctx, configFile, string(data))
+		timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+		_, err = globalEtcdClient.Put(timeoutCtx, configFile, string(data))
 		defer cancel()
 		return err
 	}
 
-	return saveConfig(objAPI, configFile, data)
-}
-
-func readConfigEtcd(configFile string) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	resp, err := globalEtcdClient.Get(ctx, configFile)
-	defer cancel()
-	if err != nil {
-		return nil, err
-	}
-	if resp.Count == 0 {
-		return nil, errConfigNotFound
-	}
-	for _, ev := range resp.Kvs {
-		if string(ev.Key) == configFile {
-			return ev.Value, nil
+	// Create a backup of the current config
+	oldData, err := readConfig(ctx, objAPI, configFile)
+	if err == nil {
+		backupConfigFile := path.Join(minioConfigPrefix, minioConfigBackupFile)
+		if err = saveConfig(ctx, objAPI, backupConfigFile, oldData); err != nil {
+			return err
+		}
+	} else {
+		if err != errConfigNotFound {
+			return err
 		}
 	}
-	return nil, errConfigNotFound
+
+	// Save the new config in the std config path
+	return saveConfig(ctx, objAPI, configFile, data)
 }
 
 func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
 	var configData []byte
 	var err error
+
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	if globalEtcdClient != nil {
-		configData, err = readConfigEtcd(configFile)
+		configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
 	} else {
-		var reader io.Reader
-		reader, err = readConfig(ctx, objAPI, configFile)
-		if err != nil {
-			return nil, err
-		}
-		configData, err = ioutil.ReadAll(reader)
+		configData, err = readConfig(ctx, objAPI, configFile)
 	}
 	if err != nil {
 		return nil, err
@@ -105,81 +98,17 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, e
 	}
 
 	var config = &serverConfig{}
-	if err := json.Unmarshal(configData, config); err != nil {
+	if err = json.Unmarshal(configData, config); err != nil {
 		return nil, err
 	}
 
-	if err := quick.CheckData(config); err != nil {
+	if err = quick.CheckData(config); err != nil {
 		return nil, err
 	}
 
 	return config, nil
 }
 
-func checkServerConfigEtcd(configFile string) error {
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-	resp, err := globalEtcdClient.Get(ctx, configFile)
-	defer cancel()
-	if err != nil {
-		return err
-	}
-	if resp.Count == 0 {
-		return errConfigNotFound
-	}
-	return nil
-}
-
-func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	if globalEtcdClient != nil {
-		return checkServerConfigEtcd(configFile)
-	}
-
-	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile); err != nil {
-		if isErrObjectNotFound(err) {
-			return errConfigNotFound
-		}
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
-		return err
-	}
-	return nil
-}
-
-func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
-	if err != nil {
-		return err
-	}
-
-	_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
-	return err
-}
-
-var errConfigNotFound = errors.New("config file not found")
-
-func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
-	var buffer bytes.Buffer
-	// Read entire content by setting size to -1
-	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, ""); err != nil {
-		// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
-			return nil, errConfigNotFound
-		}
-
-		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
-		logger.LogIf(ctx, err)
-		return nil, err
-	}
-
-	// Return config not found on empty content.
-	if buffer.Len() == 0 {
-		return nil, errConfigNotFound
-	}
-
-	return &buffer, nil
-}
-
 // ConfigSys - config system.
 type ConfigSys struct{}
 
@@ -193,7 +122,33 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errInvalidArgument
 	}
-	return initConfig(objAPI)
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing configuration needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	//  - Write quorum not met when upgrading configuration
+	//    version is needed.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			err := initConfig(objAPI)
+			if err != nil {
+				if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+					strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+					logger.Info("Waiting for configuration to be initialized..")
+					continue
+				}
+				return err
+			}
+
+			return nil
+		}
+	}
 }
 
 // NewConfigSys - creates new config system object.
@@ -202,18 +157,27 @@ func NewConfigSys() *ConfigSys {
 }
 
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
-func migrateConfigToMinioSys() error {
+func migrateConfigToMinioSys(objAPI ObjectLayer) error {
+	defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
+
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	// Verify if backend already has the file.
-	if err := checkServerConfig(context.Background(), newObjectLayerFn()); err != errConfigNotFound {
+	if err := checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
 	var config = &serverConfig{}
 	if _, err := Load(getConfigFile(), config); err != nil {
-		return err
+		if !os.IsNotExist(err) {
+			return err
+		}
+		// Read from deprecate file as well if necessary.
+		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
+			return err
+		}
 	}
 
-	return saveServerConfig(newObjectLayerFn(), config)
+	return saveServerConfig(context.Background(), objAPI, config)
 }
 
 // Initialize and load config from remote etcd or local config directory
@@ -227,25 +191,38 @@ func initConfig(objAPI ObjectLayer) error {
 		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
 		cancel()
 		if err == nil && resp.Count > 0 {
-			return migrateConfig()
+			if err = migrateConfig(); err != nil {
+				return err
+			}
+
+			// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
+			if err := migrateConfigToMinioSys(objAPI); err != nil {
+				return err
+			}
 		}
 	} else {
 		if isFile(getConfigFile()) {
 			if err := migrateConfig(); err != nil {
 				return err
 			}
-
-			// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
-			if err := migrateConfigToMinioSys(); err != nil {
-				return err
-			}
+		}
+		// Migrates ${HOME}/.minio/config.json or config.json.deprecated
+		// to '<export_path>/.minio.sys/config/config.json'
+		// ignore if the file doesn't exist.
+		if err := migrateConfigToMinioSys(objAPI); err != nil && !os.IsNotExist(err) {
+			return err
 		}
 	}
 
-	if err := checkServerConfig(context.Background(), objAPI); err != nil {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Watch config for changes and reloads them in-memory.
+	go watchConfig(objAPI, configFile, loadConfig)
+
+	if err := checkConfig(context.Background(), objAPI, configFile); err != nil {
 		if err == errConfigNotFound {
 			// Config file does not exist, we create it fresh and return upon success.
-			if err = newConfig(objAPI); err != nil {
+			if err = newSrvConfig(objAPI); err != nil {
 				return err
 			}
 		} else {

cmd/copy-part-range.go

@@ -17,11 +17,8 @@
 package cmd
 
 import (
-	"fmt"
 	"net/http"
 	"net/url"
-	"strconv"
-	"strings"
 )
 
 // Writes S3 compatible copy part range error.
@@ -39,68 +36,33 @@ func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL) {
 	}
 }
 
-// Parses x-amz-copy-source-range for CopyObjectPart API. Specifically written to
-// differentiate the behavior between regular httpRange header v/s x-amz-copy-source-range.
-// The range of bytes to copy from the source object. The range value must use the form
-// bytes=first-last, where the first and last are the zero-based byte offsets to copy.
-// For example, bytes=0-9 indicates that you want to copy the first ten bytes of the source.
+// Parses x-amz-copy-source-range for CopyObjectPart API. Its behavior
+// is different from regular HTTP range header. It only supports the
+// form `bytes=first-last` where first and last are zero-based byte
+// offsets. See
 // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html
-func parseCopyPartRange(rangeString string, resourceSize int64) (hrange *httpRange, err error) {
-	// Return error if given range string doesn't start with byte range prefix.
-	if !strings.HasPrefix(rangeString, byteRangePrefix) {
-		return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix)
+// for full details. This function treats an empty rangeString as
+// referring to the whole resource.
+func parseCopyPartRangeSpec(rangeString string) (hrange *HTTPRangeSpec, err error) {
+	hrange, err = parseRequestRangeSpec(rangeString)
+	if err != nil {
+		return nil, err
 	}
-
-	// Trim byte range prefix.
-	byteRangeString := strings.TrimPrefix(rangeString, byteRangePrefix)
-
-	// Check if range string contains delimiter '-', else return error. eg. "bytes=8"
-	sepIndex := strings.Index(byteRangeString, "-")
-	if sepIndex == -1 {
+	if hrange.IsSuffixLength || hrange.Start < 0 || hrange.End < 0 {
 		return nil, errInvalidRange
 	}
-
-	offsetBeginString := byteRangeString[:sepIndex]
-	offsetBegin := int64(-1)
-	// Convert offsetBeginString only if its not empty.
-	if len(offsetBeginString) > 0 {
-		if !validBytePos.MatchString(offsetBeginString) {
-			return nil, errInvalidRange
-		}
-		if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil {
-			return nil, errInvalidRange
-		}
-	}
-
-	offsetEndString := byteRangeString[sepIndex+1:]
-	offsetEnd := int64(-1)
-	// Convert offsetEndString only if its not empty.
-	if len(offsetEndString) > 0 {
-		if !validBytePos.MatchString(offsetEndString) {
-			return nil, errInvalidRange
-		}
-		if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil {
-			return nil, errInvalidRange
-		}
-	}
-
-	// rangeString contains first byte positions. eg. "bytes=2-" or
-	// rangeString contains last bye positions. eg. "bytes=-2"
-	if offsetBegin == -1 || offsetEnd == -1 {
-		return nil, errInvalidRange
-	}
-
-	// Last byte position should not be greater than first byte
-	// position. eg. "bytes=5-2"
-	if offsetBegin > offsetEnd {
-		return nil, errInvalidRange
-	}
-
-	// First and last byte positions should not be >= resourceSize.
-	if offsetBegin >= resourceSize || offsetEnd >= resourceSize {
-		return nil, errInvalidRangeSource
-	}
-
-	// Success..
-	return &httpRange{offsetBegin, offsetEnd, resourceSize}, nil
+	return hrange, nil
+}
+
+// checkCopyPartRangeWithSize adds more check to the range string in case of
+// copy object part. This API requires having specific start and end  range values
+// e.g. 'bytes=3-10'. Other use cases will be rejected.
+func checkCopyPartRangeWithSize(rs *HTTPRangeSpec, resourceSize int64) (err error) {
+	if rs == nil {
+		return nil
+	}
+	if rs.IsSuffixLength || rs.Start >= resourceSize || rs.End >= resourceSize {
+		return errInvalidRangeSource
+	}
+	return nil
 }

cmd/copy-part-range_test.go

@@ -19,35 +19,37 @@ package cmd
 import "testing"
 
 // Test parseCopyPartRange()
-func TestParseCopyPartRange(t *testing.T) {
+func TestParseCopyPartRangeSpec(t *testing.T) {
 	// Test success cases.
 	successCases := []struct {
 		rangeString string
 		offsetBegin int64
 		offsetEnd   int64
-		length      int64
 	}{
-		{"bytes=2-5", 2, 5, 4},
-		{"bytes=2-9", 2, 9, 8},
-		{"bytes=2-2", 2, 2, 1},
-		{"bytes=0000-0006", 0, 6, 7},
+		{"bytes=2-5", 2, 5},
+		{"bytes=2-9", 2, 9},
+		{"bytes=2-2", 2, 2},
+		{"bytes=0000-0006", 0, 6},
 	}
+	objectSize := int64(10)
 
 	for _, successCase := range successCases {
-		hrange, err := parseCopyPartRange(successCase.rangeString, 10)
+		rs, err := parseCopyPartRangeSpec(successCase.rangeString)
 		if err != nil {
 			t.Fatalf("expected: <nil>, got: %s", err)
 		}
 
-		if hrange.offsetBegin != successCase.offsetBegin {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, hrange.offsetBegin)
+		start, length, err1 := rs.GetOffsetLength(objectSize)
+		if err1 != nil {
+			t.Fatalf("expected: <nil>, got: %s", err1)
 		}
 
-		if hrange.offsetEnd != successCase.offsetEnd {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, hrange.offsetEnd)
+		if start != successCase.offsetBegin {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, start)
 		}
-		if hrange.getLength() != successCase.length {
-			t.Fatalf("expected: %d, got: %d", successCase.length, hrange.getLength())
+
+		if start+length-1 != successCase.offsetEnd {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, start+length-1)
 		}
 	}
 
@@ -59,15 +61,16 @@ func TestParseCopyPartRange(t *testing.T) {
 		"bytes=2-+5",
 		"bytes=2--5",
 		"bytes=-",
-		"",
 		"2-5",
 		"bytes = 2-5",
 		"bytes=2 - 5",
 		"bytes=0-0,-1",
 		"bytes=2-5 ",
+		"bytes=-1",
+		"bytes=1-",
 	}
 	for _, rangeString := range invalidRangeStrings {
-		if _, err := parseCopyPartRange(rangeString, 10); err == nil {
+		if _, err := parseCopyPartRangeSpec(rangeString); err == nil {
 			t.Fatalf("expected: an error, got: <nil> for range %s", rangeString)
 		}
 	}
@@ -78,8 +81,14 @@ func TestParseCopyPartRange(t *testing.T) {
 		"bytes=20-30",
 	}
 	for _, rangeString := range errorRangeString {
-		if _, err := parseCopyPartRange(rangeString, 10); err != errInvalidRangeSource {
-			t.Fatalf("expected: %s, got: %s", errInvalidRangeSource, err)
+		rs, err := parseCopyPartRangeSpec(rangeString)
+		if err == nil {
+			err1 := checkCopyPartRangeWithSize(rs, objectSize)
+			if err1 != errInvalidRangeSource {
+				t.Fatalf("expected: %s, got: %s", errInvalidRangeSource, err)
+			}
+		} else {
+			t.Fatalf("expected: %s, got: <nil>", errInvalidRangeSource)
 		}
 	}
 }

cmd/crypto/error.go

@@ -43,6 +43,10 @@ var (
 	// base64-encoded string or not 256 bits long.
 	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
 
+	// ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key)
+	// does not match the secret key used during encrypting the object.
+	ErrSecretKeyMismatch = errors.New("The secret key does not match the secret key used during upload")
+
 	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
 	// computed MD5 sum. This means that the client provided either the wrong key for
 	// a certain MD5 checksum or the wrong MD5 for a certain key.

cmd/crypto/header.go

@@ -74,6 +74,14 @@ const (
 	SSEAlgorithmKMS = "aws:kms"
 )
 
+// RemoveSensitiveHeaders removes confidential encryption
+// information - e.g. the SSE-C key - from the HTTP headers.
+// It has the same semantics as RemoveSensitiveEntires.
+func RemoveSensitiveHeaders(h http.Header) {
+	h.Del(SSECKey)
+	h.Del(SSECopyKey)
+}
+
 // S3 represents AWS SSE-S3. It provides functionality to handle
 // SSE-S3 requests.
 var S3 = s3{}
@@ -164,7 +172,6 @@ func (ssecCopy) IsRequested(h http.Header) bool {
 // ParseHTTP parses the SSE-C headers and returns the SSE-C client key
 // on success. SSE-C copy headers are ignored.
 func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	defer h.Del(SSECKey) // remove SSE-C key from headers after parsing
 	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
 		return key, ErrInvalidCustomerAlgorithm
 	}
@@ -190,7 +197,6 @@ func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
 // ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
 // on success. Regular SSE-C headers are ignored.
 func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	defer h.Del(SSECopyKey) // remove SSE-C copy key of source object from headers after parsing
 	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
 		return key, ErrInvalidCustomerAlgorithm
 	}

cmd/crypto/header_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"net/http"
+	"sort"
 	"testing"
 )
 
@@ -250,9 +251,6 @@ func TestSSECParse(t *testing.T) {
 		if err == nil && key == zeroKey {
 			t.Errorf("Test %d: parsed client key is zero key", i)
 		}
-		if _, ok := test.Header[SSECKey]; ok {
-			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
-		}
 	}
 }
 
@@ -337,3 +335,102 @@ func TestSSECopyParse(t *testing.T) {
 		}
 	}
 }
+
+var removeSensitiveHeadersTests = []struct {
+	Header, ExpectedHeader http.Header
+}{
+	{
+		Header: http.Header{
+			SSECKey:    []string{""},
+			SSECopyKey: []string{""},
+		},
+		ExpectedHeader: http.Header{},
+	},
+	{ // Standard SSE-C request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + SSE-C-copy request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + metadata request headers
+		Header: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKey:             []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+	},
+}
+
+func TestRemoveSensitiveHeaders(t *testing.T) {
+	isEqual := func(x, y http.Header) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			u, ok := y[k]
+			if !ok || len(v) != len(u) {
+				return false
+			}
+			sort.Strings(v)
+			sort.Strings(u)
+			for j := range v {
+				if v[j] != u[j] {
+					return false
+				}
+			}
+		}
+		return true
+	}
+	areKeysEqual := func(h http.Header, metadata map[string]string) bool {
+		if len(h) != len(metadata) {
+			return false
+		}
+		for k := range h {
+			if _, ok := metadata[k]; !ok {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeSensitiveHeadersTests {
+		metadata := make(map[string]string, len(test.Header))
+		for k := range test.Header {
+			metadata[k] = "" // set metadata key - we don't care about the value
+		}
+
+		RemoveSensitiveHeaders(test.Header)
+		if !isEqual(test.ExpectedHeader, test.Header) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+		RemoveSensitiveEntries(metadata)
+		if !areKeysEqual(test.ExpectedHeader, metadata) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+	}
+}

cmd/crypto/key.go

@@ -124,7 +124,7 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 	}
 
 	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
-		return err // TODO(aead): upgrade sio to use sio.Error
+		return ErrSecretKeyMismatch
 	}
 	copy(key[:], decryptedKey.Bytes())
 	return nil
@@ -140,3 +140,37 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
 	mac.Sum(partKey[:0])
 	return partKey
 }
+
+// SealETag seals the etag using the object key.
+// It does not encrypt empty ETags because such ETags indicate
+// that the S3 client hasn't sent an ETag = MD5(object) and
+// the backend can pick an ETag value.
+func (key ObjectKey) SealETag(etag []byte) []byte {
+	if len(etag) == 0 { // don't encrypt empty ETag - only if client sent ETag = MD5(object)
+		return etag
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Encrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		logger.CriticalIf(context.Background(), errors.New("Unable to encrypt ETag using object key"))
+	}
+	return buffer.Bytes()
+}
+
+// UnsealETag unseals the etag using the provided object key.
+// It does not try to decrypt the ETag if len(etag) == 16
+// because such ETags indicate that the S3 client hasn't sent
+// an ETag = MD5(object) and the backend has picked an ETag value.
+func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
+	if !IsETagSealed(etag) {
+		return etag, nil
+	}
+	var buffer bytes.Buffer
+	mac := hmac.New(sha256.New, key[:])
+	mac.Write([]byte("SSE-etag"))
+	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
+		return nil, err
+	}
+	return buffer.Bytes(), nil
+}

cmd/crypto/key_test.go

@@ -166,3 +166,31 @@ func TestDerivePartKey(t *testing.T) {
 		}
 	}
 }
+
+var sealUnsealETagTests = []string{
+	"",
+	"90682b8e8cc7609c",
+	"90682b8e8cc7609c4671e1d64c73fc30",
+	"90682b8e8cc7609c4671e1d64c73fc307fb3104f",
+}
+
+func TestSealETag(t *testing.T) {
+	var key ObjectKey
+	for i := range key {
+		key[i] = byte(i)
+	}
+	for i, etag := range sealUnsealETagTests {
+		tag, err := hex.DecodeString(etag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		sealedETag := key.SealETag(tag)
+		unsealedETag, err := key.UnsealETag(sealedETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decrypt etag: %s", i, err)
+		}
+		if !bytes.Equal(unsealedETag, tag) {
+			t.Errorf("Test %d: unsealed etag does not match: got %s - want %s", i, hex.EncodeToString(unsealedETag), etag)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -32,6 +32,26 @@ func IsMultiPart(metadata map[string]string) bool {
 	return false
 }
 
+// RemoveSensitiveEntries removes confidential encryption
+// information - e.g. the SSE-C key - from the metadata map.
+// It has the same semantics as RemoveSensitiveHeaders.
+func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
+	delete(metadata, SSECKey)
+	delete(metadata, SSECopyKey)
+}
+
+// RemoveInternalEntries removes all crypto-specific internal
+// metadata entries from the metadata map.
+func RemoveInternalEntries(metadata map[string]string) {
+	delete(metadata, SSEMultipart)
+	delete(metadata, SSEIV)
+	delete(metadata, SSESealAlgorithm)
+	delete(metadata, SSECSealedKey)
+	delete(metadata, S3SealedKey)
+	delete(metadata, S3KMSKeyID)
+	delete(metadata, S3KMSSealedKey)
+}
+
 // IsEncrypted returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption.
 //
@@ -211,3 +231,6 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
 	copy(sealedKey.Key[:], encryptedKey)
 	return sealedKey, nil
 }
+
+// IsETagSealed returns true if the etag seems to be encrypted.
+func IsETagSealed(etag []byte) bool { return len(etag) > 16 }

cmd/crypto/metadata_test.go

@@ -17,6 +17,7 @@ package crypto
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/hex"
 	"testing"
 
 	"github.com/minio/minio/cmd/logger"
@@ -326,15 +327,20 @@ func TestS3CreateMetadata(t *testing.T) {
 	_ = S3.CreateMetadata(nil, "", []byte{}, SealedKey{Algorithm: InsecureSealAlgorithm})
 }
 
-var ssecCreateMetadataTests = []SealedKey{
-	{Algorithm: SealAlgorithm},
-	{IV: [32]byte{0xff}, Key: [64]byte{0x7e}, Algorithm: SealAlgorithm},
+var ssecCreateMetadataTests = []struct {
+	KeyID         string
+	SealedDataKey []byte
+	SealedKey     SealedKey
+}{
+	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
 }
 
 func TestSSECCreateMetadata(t *testing.T) {
 	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
 	logger.Disable = true
-	for i, test := range s3CreateMetadataTests {
+	for i, test := range ssecCreateMetadataTests {
 		metadata := SSEC.CreateMetadata(nil, test.SealedKey)
 		sealedKey, err := SSEC.ParseMetadata(metadata)
 		if err != nil {
@@ -359,3 +365,75 @@ func TestSSECCreateMetadata(t *testing.T) {
 	}()
 	_ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm})
 }
+
+var isETagSealedTests = []struct {
+	ETag     string
+	IsSealed bool
+}{
+	{ETag: "", IsSealed: false},                                                                                                // 0
+	{ETag: "90682b8e8cc7609c4671e1d64c73fc30", IsSealed: false},                                                                // 1
+	{ETag: "f201040c9dc593e39ea004dc1323699bcd", IsSealed: true},                                                               // 2 not valid ciphertext but looks like sealed ETag
+	{ETag: "20000f00fba2ee2ae4845f725964eeb9e092edfabc7ab9f9239e8344341f769a51ce99b4801b0699b92b16a72fa94972", IsSealed: true}, // 3
+}
+
+func TestIsETagSealed(t *testing.T) {
+	for i, test := range isETagSealedTests {
+		etag, err := hex.DecodeString(test.ETag)
+		if err != nil {
+			t.Errorf("Test %d: failed to decode etag: %s", i, err)
+		}
+		if sealed := IsETagSealed(etag); sealed != test.IsSealed {
+			t.Errorf("Test %d: got %v - want %v", i, sealed, test.IsSealed)
+		}
+	}
+}
+
+var removeInternalEntriesTests = []struct {
+	Metadata, Expected map[string]string
+}{
+	{ // 0
+		Metadata: map[string]string{
+			SSEMultipart:     "",
+			SSEIV:            "",
+			SSESealAlgorithm: "",
+			SSECSealedKey:    "",
+			S3SealedKey:      "",
+			S3KMSKeyID:       "",
+			S3KMSSealedKey:   "",
+		},
+		Expected: map[string]string{},
+	},
+	{ // 1
+		Metadata: map[string]string{
+			SSEMultipart:         "",
+			SSEIV:                "",
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+		Expected: map[string]string{
+			"X-Amz-Meta-A":       "X",
+			"X-Minio-Internal-B": "Y",
+		},
+	},
+}
+
+func TestRemoveInternalEntries(t *testing.T) {
+	isEqual := func(x, y map[string]string) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			if u, ok := y[k]; !ok || v != u {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeInternalEntriesTests {
+		RemoveInternalEntries(test.Metadata)
+		if !isEqual(test.Metadata, test.Expected) {
+			t.Errorf("Test %d: got %v - want %v", i, test.Metadata, test.Expected)
+		}
+	}
+}

cmd/crypto/sse.go

@@ -18,6 +18,8 @@ import (
 	"context"
 	"errors"
 	"io"
+	"net/http"
+	"path"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/ioutil"
@@ -69,10 +71,59 @@ const (
 // domain is "SSE-S3".
 func (s3) String() string { return "SSE-S3" }
 
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using KMS and returns the decrypted object
+// key.
+func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if err != nil {
+		return
+	}
+	err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
+	return
+}
+
 // String returns the SSE domain as string. For SSE-C the
 // domain is "SSE-C".
 func (ssec) String() string { return "SSE-C" }
 
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-C client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// UnsealObjectKey extracts and decrypts the sealed object key
+// from the metadata using the SSE-Copy client key of the HTTP headers
+// and returns the decrypted object key.
+func (sse ssecCopy) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	clientKey, err := sse.ParseHTTP(h)
+	if err != nil {
+		return
+	}
+	return unsealObjectKey(clientKey, metadata, bucket, object)
+}
+
+// unsealObjectKey decrypts and returns the sealed object key
+// from the metadata using the SSE-C client key.
+func unsealObjectKey(clientKey [32]byte, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
+	sealedKey, err := SSEC.ParseMetadata(metadata)
+	if err != nil {
+		return
+	}
+	err = key.Unseal(clientKey, sealedKey, SSEC.String(), bucket, object)
+	return
+}
+
 // EncryptSinglePart encrypts an io.Reader which must be the
 // the body of a single-part PUT request.
 func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {

cmd/crypto/sse_test.go

@@ -14,7 +14,10 @@
 
 package crypto
 
-import "testing"
+import (
+	"net/http"
+	"testing"
+)
 
 func TestS3String(t *testing.T) {
 	const Domain = "SSE-S3"
@@ -29,3 +32,195 @@ func TestSSECString(t *testing.T) {
 		t.Errorf("SSEC's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
 	}
 }
+
+var ssecUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECUnsealObjectKey(t *testing.T) {
+	for i, test := range ssecUnsealObjectKeyTests {
+		if _, err := SSEC.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var sseCopyUnsealObjectKeyTests = []struct {
+	Headers        http.Header
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object2",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrSecretKeyMismatch,
+	},
+	{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5":   []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":         "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+	{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
+		Headers: http.Header{
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
+			"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key":       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+		},
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Sealed-Key":     "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
+			"X-Minio-Internal-Server-Side-Encryption-Iv":             "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
+		},
+		ExpectedErr: ErrMissingCustomerKeyMD5,
+	},
+}
+
+func TestSSECopyUnsealObjectKey(t *testing.T) {
+	for i, test := range sseCopyUnsealObjectKeyTests {
+		if _, err := SSECopy.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}
+
+var s3UnsealObjectKeyTests = []struct {
+	KMS            KMS
+	Bucket, Object string
+	Metadata       map[string]string
+
+	ExpectedErr error
+}{
+	{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+			"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":    "DAREv2-HMAC-SHA256",
+		},
+		ExpectedErr: nil,
+	},
+	{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
+		KMS:    NewKMS([32]byte{}),
+		Bucket: "bucket",
+		Object: "object",
+		Metadata: map[string]string{
+			"X-Minio-Internal-Server-Side-Encryption-Iv":                "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":     "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
+			"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":     "test-key-1",
+		},
+		ExpectedErr: errMissingInternalSealAlgorithm,
+	},
+}
+
+func TestS3UnsealObjectKey(t *testing.T) {
+	for i, test := range s3UnsealObjectKeyTests {
+		if _, err := S3.UnsealObjectKey(test.KMS, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
+			t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
+		}
+	}
+}

cmd/crypto/vault.go

@@ -28,8 +28,8 @@ import (
 )
 
 const (
-	// VaultEndpointEnv Vault endpoint environment variable
-	VaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
+	// vaultEndpointEnv Vault endpoint environment variable
+	vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
 	// vaultAuthTypeEnv type of vault auth to be used
 	vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
 	// vaultAppRoleIDEnv  Vault AppRole ID environment variable
@@ -40,6 +40,10 @@ const (
 	vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
 	// vaultKeyNameEnv Vault Encryption Key Name environment variable
 	vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
+
+	// vaultCAPath is the path to a directory of PEM-encoded CA
+	// cert files to verify the Vault server SSL certificate.
+	vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
 )
 
 var (
@@ -93,7 +97,7 @@ type VaultConfig struct {
 // been set
 func validateVaultConfig(c *VaultConfig) error {
 	if c.Endpoint == "" {
-		return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", VaultEndpointEnv)
+		return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
 	}
 	if strings.ToLower(c.Auth.Type) != "approle" {
 		return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
@@ -110,7 +114,6 @@ func validateVaultConfig(c *VaultConfig) error {
 	if c.Key.Version < 0 {
 		return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
 	}
-
 	return nil
 }
 
@@ -134,7 +137,7 @@ func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (tok
 // variables and performs validations.
 func NewVaultConfig() (KMSConfig, error) {
 	kc := KMSConfig{}
-	endpoint := os.Getenv(VaultEndpointEnv)
+	endpoint := os.Getenv(vaultEndpointEnv)
 	roleID := os.Getenv(vaultAppRoleIDEnv)
 	roleSecret := os.Getenv(vaultAppSecretIDEnv)
 	keyName := os.Getenv(vaultKeyNameEnv)
@@ -177,9 +180,15 @@ func NewVaultConfig() (KMSConfig, error) {
 // and gets a client token for future api calls.
 func NewVault(kmsConf KMSConfig) (KMS, error) {
 	config := kmsConf.Vault
-	c, err := vault.NewClient(&vault.Config{
+	vconfig := &vault.Config{
 		Address: config.Endpoint,
-	})
+	}
+	if err := vconfig.ConfigureTLS(&vault.TLSConfig{
+		CAPath: os.Getenv(vaultCAPath),
+	}); err != nil {
+		return nil, err
+	}
+	c, err := vault.NewClient(vconfig)
 	if err != nil {
 		return nil, err
 	}

cmd/disk-cache-fs.go

@@ -258,7 +258,7 @@ func (cfs *cacheFSObjects) IsOnline() bool {
 }
 
 // Caches the object to disk
-func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) error {
+func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) error {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -275,7 +275,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 			return pErr
 		}
 	}
-	_, err := cfs.PutObject(ctx, bucket, object, data, metadata)
+	_, err := cfs.PutObject(ctx, bucket, object, data, metadata, opts)
 	// if err is due to disk being offline , mark cache drive as offline
 	if IsErr(err, baseErrs...) {
 		cfs.setOnline(false)
@@ -284,8 +284,8 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 }
 
 // Returns the handle for the cached object
-func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
-	return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag)
+func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
+	return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
 }
 
 // Deletes the cached object
@@ -295,13 +295,13 @@ func (cfs *cacheFSObjects) Delete(ctx context.Context, bucket, object string) (e
 
 // convenience function to check if object is cached on this cacheFSObjects
 func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bool {
-	_, err := cfs.GetObjectInfo(ctx, bucket, object)
+	_, err := cfs.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	return err == nil
 }
 
 // Identical to fs PutObject operation except that it uses ETag in metadata
 // headers.
-func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	fs := cfs.FSObjects
 	// Lock the object.
 	objectLock := fs.nsMutex.NewNSLock(bucket, object)
@@ -354,7 +354,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	}
 
 	// Validate input data size and it can never be less than zero.
-	if data.Size() < 0 {
+	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return ObjectInfo{}, errInvalidArgument
 	}
@@ -438,7 +438,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 // Implements S3 compatible initiate multipart API. Operation here is identical
 // to fs backend implementation - with the exception that cache FS uses the uploadID
 // generated on the backend
-func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string) (string, error) {
+func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string, opts ObjectOptions) (string, error) {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:

cmd/disk-cache.go

@@ -31,14 +31,14 @@ import (
 
 	"github.com/djherbis/atime"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
-	// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
-	cacheSizeMultiplier  = 100
+	// disk cache needs to have  object size space free for a cache entry to be created.
 	cacheTrashDir        = "trash"
 	cacheCleanupInterval = 10 // in minutes
 )
@@ -57,16 +57,17 @@ type cacheObjects struct {
 	// file path patterns to exclude from cache
 	exclude []string
 	// Object functions pointing to the corresponding functions of backend implementation.
-	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
-	GetObjectInfoFn           func(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error)
-	PutObjectFn               func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
+	GetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObjectFn               func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn            func(ctx context.Context, bucket, object string) error
 	ListObjectsFn             func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2Fn           func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListBucketsFn             func(ctx context.Context) (buckets []BucketInfo, err error)
 	GetBucketInfoFn           func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
-	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error)
-	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
+	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
 	DeleteBucketFn            func(ctx context.Context, bucket string) error
@@ -88,14 +89,15 @@ type CacheObjectLayer interface {
 	ListBuckets(ctx context.Context) (buckets []BucketInfo, err error)
 	DeleteBucket(ctx context.Context, bucket string) error
 	// Object operations.
-	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
-	GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
-	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error)
-	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
 
@@ -103,6 +105,11 @@ type CacheObjectLayer interface {
 	StorageInfo(ctx context.Context) CacheStorageInfo
 }
 
+// IsCacheable returns if the object should be saved in the cache.
+func (o ObjectInfo) IsCacheable() bool {
+	return !crypto.IsEncrypted(o.UserDefined)
+}
+
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
 func backendDownError(err error) bool {
 	_, backendDown := err.(BackendDown)
@@ -175,22 +182,99 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string {
 	return metadata
 }
 
+func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	if c.isCacheExclude(bucket, object) {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
+	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
+	if err != nil {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+
+	objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
+	if backendDownError(err) && cacheErr == nil {
+		return cacheReader, nil
+	} else if err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			// Delete cached entry if backend object was deleted.
+			dcache.Delete(ctx, bucket, object)
+		}
+		return nil, err
+	}
+
+	if !objInfo.IsCacheable() || filterFromCache(objInfo.UserDefined) {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	if cacheErr == nil {
+		if cacheReader.ObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
+			// Object is not stale, so serve from cache
+			return cacheReader, nil
+		}
+		cacheReader.Close()
+		// Object is stale, so delete from cache
+		dcache.Delete(ctx, bucket, object)
+	}
+
+	// Since we got here, we are serving the request from backend,
+	// and also adding the object to the cache.
+
+	if rs != nil {
+		// We don't cache partial objects.
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+	if !dcache.diskAvailable(objInfo.Size) {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	if bkErr != nil {
+		return nil, bkErr
+	}
+
+	// Initialize pipe.
+	pipeReader, pipeWriter := io.Pipe()
+	teeReader := io.TeeReader(bkReader, pipeWriter)
+	hashReader, herr := hash.NewReader(pipeReader, bkReader.ObjInfo.Size, "", "", bkReader.ObjInfo.Size)
+	if herr != nil {
+		bkReader.Close()
+		return nil, herr
+	}
+
+	go func() {
+		opts := ObjectOptions{}
+		putErr := dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(bkReader.ObjInfo), opts)
+		// close the write end of the pipe, so the error gets
+		// propagated to getObjReader
+		pipeWriter.CloseWithError(putErr)
+	}()
+
+	cleanupBackend := func() { bkReader.Close() }
+	cleanupPipe := func() { pipeReader.Close() }
+	gr = NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, cleanupBackend, cleanupPipe)
+	return gr, nil
+}
+
 // Uses cached-object to serve the request. If object is not cached it serves the request from the backend and also
 // stores it in the cache for serving subsequent requests.
-func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	GetObjectFn := c.GetObjectFn
 	GetObjectInfoFn := c.GetObjectInfoFn
 
 	if c.isCacheExclude(bucket, object) {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
 	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
 	if err != nil {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// stat object on backend
-	objInfo, err := GetObjectInfoFn(ctx, bucket, object)
+	objInfo, err := GetObjectInfoFn(ctx, bucket, object, opts)
 	backendDown := backendDownError(err)
 	if err != nil && !backendDown {
 		if _, ok := err.(ObjectNotFound); ok {
@@ -200,43 +284,43 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		return err
 	}
 
-	if !backendDown && filterFromCache(objInfo.UserDefined) {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+	if !backendDown && !objInfo.IsCacheable() {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 
-	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object)
+	if !backendDown && filterFromCache(objInfo.UserDefined) {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
+	}
+
+	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts)
 	if err == nil {
 		if backendDown {
 			// If the backend is down, serve the request from cache.
-			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag)
+			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		}
 		if cachedObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
-			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag)
+			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		}
 		dcache.Delete(ctx, bucket, object)
 	}
 	if startOffset != 0 || length != objInfo.Size {
 		// We don't cache partial objects.
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
-	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
-		// cache only objects < 1/100th of disk capacity
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+	if !dcache.diskAvailable(objInfo.Size) {
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "")
+	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "", objInfo.Size)
 	if err != nil {
 		return err
 	}
 	go func() {
-		if err = GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag); err != nil {
-			pipeWriter.CloseWithError(err)
-			return
-		}
-		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
+		gerr := GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts)
+		pipeWriter.CloseWithError(gerr) // Close writer explicitly signaling we wrote all data.
 	}()
-	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
+	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo), opts)
 	if err != nil {
 		return err
 	}
@@ -245,17 +329,17 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 }
 
 // Returns ObjectInfo from cache if available.
-func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string) (ObjectInfo, error) {
+func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 	getObjectInfoFn := c.GetObjectInfoFn
 	if c.isCacheExclude(bucket, object) {
-		return getObjectInfoFn(ctx, bucket, object)
+		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
 	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
 	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
 	if err != nil {
-		return getObjectInfoFn(ctx, bucket, object)
+		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
-	objInfo, err := getObjectInfoFn(ctx, bucket, object)
+	objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
 	if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
@@ -266,14 +350,14 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 			return ObjectInfo{}, err
 		}
 		// when backend is down, serve from cache.
-		cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object)
+		cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object, opts)
 		if cerr == nil {
 			return cachedObjInfo, nil
 		}
 		return ObjectInfo{}, BackendDown{}
 	}
 	// when backend is up, do a sanity check on cached object
-	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object)
+	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		return objInfo, nil
 	}
@@ -560,42 +644,42 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool {
 }
 
 // PutObject - caches the uploaded object for single Put operations
-func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	putObjectFn := c.PutObjectFn
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectFn(ctx, bucket, object, r, metadata)
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	size := r.Size()
 
 	// fetch from backend if there is no space on cache drive
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
-		return putObjectFn(ctx, bucket, object, r, metadata)
+	if !dcache.diskAvailable(size) {
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	// fetch from backend if cache exclude pattern or cache-control
 	// directive set to exclude
 	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
 		dcache.Delete(ctx, bucket, object)
-		return putObjectFn(ctx, bucket, object, r, metadata)
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, r.MD5HexString(), r.SHA256HexString())
+	hashReader, err := hash.NewReader(pipeReader, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, r.MD5HexString(), r.SHA256HexString())
+	cHashReader, err := hash.NewReader(rPipe, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	oinfoCh := make(chan ObjectInfo)
 	errCh := make(chan error)
 	go func() {
-		oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata)
+		oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata, opts)
 		if perr != nil {
 			pipeWriter.CloseWithError(perr)
 			wPipe.CloseWithError(perr)
@@ -608,7 +692,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	}()
 
 	go func() {
-		if err = dcache.Put(ctx, bucket, object, cHashReader, metadata); err != nil {
+		if err = dcache.Put(ctx, bucket, object, cHashReader, metadata, opts); err != nil {
 			wPipe.CloseWithError(err)
 			return
 		}
@@ -627,68 +711,68 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 }
 
 // NewMultipartUpload - Starts a new multipart upload operation to backend and cache.
-func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	newMultipartUploadFn := c.NewMultipartUploadFn
 
 	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
-		return newMultipartUploadFn(ctx, bucket, object, metadata)
+		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	}
 
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return newMultipartUploadFn(ctx, bucket, object, metadata)
+		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	}
 
-	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata)
+	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	if err != nil {
 		return
 	}
 	// create new multipart upload in cache with same uploadID
-	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID)
+	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID, opts)
 	return uploadID, err
 }
 
 // PutObjectPart - uploads part to backend and cache simultaneously.
-func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
 	putObjectPartFn := c.PutObjectPartFn
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
 	if c.isCacheExclude(bucket, object) {
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
-	// make sure cache has at least cacheSizeMultiplier * size available
+	// make sure cache has at least size space available
 	size := data.Size()
-	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
+	if !dcache.diskAvailable(size) {
 		select {
 		case dcache.purgeChan <- struct{}{}:
 		default:
 		}
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
 	info = PartInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString())
+	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString())
+	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return
 	}
 	pinfoCh := make(chan PartInfo)
 	errorCh := make(chan error)
 	go func() {
-		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader)
+		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader, opts)
 		if err != nil {
 			close(pinfoCh)
 			pipeWriter.CloseWithError(err)
@@ -700,7 +784,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		pinfoCh <- info
 	}()
 	go func() {
-		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader); perr != nil {
+		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader, opts); perr != nil {
 			wPipe.CloseWithError(perr)
 			return
 		}
@@ -876,14 +960,17 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		cache:    dcache,
 		exclude:  config.Exclude,
 		listPool: newTreeWalkPool(globalLookupTimeout),
-		GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
-			return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag)
+		GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
+			return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		},
-		GetObjectInfoFn: func(ctx context.Context, bucket, object string) (ObjectInfo, error) {
-			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object)
+		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
-			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata)
+		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+		},
+		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
 		},
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
@@ -900,11 +987,11 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetBucketInfoFn: func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
 			return newObjectLayerFn().GetBucketInfo(ctx, bucket)
 		},
-		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
-			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata)
+		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
+			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts)
 		},
-		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
-			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data)
+		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
+			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 		},
 		AbortMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string) error {
 			return newObjectLayerFn().AbortMultipartUpload(ctx, bucket, object, uploadID)

cmd/disk-cache_test.go

@@ -192,17 +192,18 @@ func TestDiskCache(t *testing.T) {
 	objInfo.ContentType = contentType
 	objInfo.ETag = etag
 	objInfo.UserDefined = httpMeta
+	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
-	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+	err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
-	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -219,7 +220,7 @@ func TestDiskCache(t *testing.T) {
 		t.Fatal("Cached content-type does not match")
 	}
 	writer := bytes.NewBuffer(nil)
-	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -266,23 +267,24 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	objInfo.ContentType = contentType
 	objInfo.ETag = etag
 	objInfo.UserDefined = httpMeta
+	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
-	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
 	if err != nil {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
 		if err != nil {
 			t.Fatal(err)
 		}
-		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -299,7 +301,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 			t.Fatal("Cached content-type does not match")
 		}
 		writer := bytes.NewBuffer(nil)
-		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/dummy-data-generator_test.go

@@ -0,0 +1,182 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+var alphabets = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
+
+// DummyDataGen returns a reader that repeats the bytes in `alphabets`
+// upto the desired length.
+type DummyDataGen struct {
+	b           []byte
+	idx, length int64
+}
+
+// NewDummyDataGen returns a ReadSeeker over the first `totalLength`
+// bytes from the infinite stream consisting of repeated
+// concatenations of `alphabets`.
+//
+// The skipOffset (generally = 0) can be used to skip a given number
+// of bytes from the beginning of the infinite stream. This is useful
+// to compare such streams of bytes that may be split up, because:
+//
+// Given the function:
+//
+// f := func(r io.Reader) string {
+//           b, _ := ioutil.ReadAll(r)
+//           return string(b)
+// }
+//
+// for example, the following is true:
+//
+// f(NewDummyDataGen(100, 0)) == f(NewDummyDataGen(50, 0)) + f(NewDummyDataGen(50, 50))
+func NewDummyDataGen(totalLength, skipOffset int64) io.ReadSeeker {
+	if totalLength < 0 {
+		panic("Negative length passed to DummyDataGen!")
+	}
+	if skipOffset < 0 {
+		panic("Negative rotations are not allowed")
+	}
+
+	skipOffset = skipOffset % int64(len(alphabets))
+	as := make([]byte, 2*len(alphabets))
+	copy(as, alphabets)
+	copy(as[len(alphabets):], alphabets)
+	b := as[skipOffset : skipOffset+int64(len(alphabets))]
+	return &DummyDataGen{
+		length: totalLength,
+		b:      b,
+	}
+}
+
+func (d *DummyDataGen) Read(b []byte) (n int, err error) {
+	k := len(b)
+	numLetters := int64(len(d.b))
+	for k > 0 && d.idx < d.length {
+		w := copy(b[len(b)-k:], d.b[d.idx%numLetters:])
+		k -= w
+		d.idx += int64(w)
+		n += w
+	}
+	if d.idx >= d.length {
+		extraBytes := d.idx - d.length
+		n -= int(extraBytes)
+		if n < 0 {
+			n = 0
+		}
+		err = io.EOF
+	}
+	return
+}
+
+func (d *DummyDataGen) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		if offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx = offset
+	case io.SeekCurrent:
+		if d.idx+offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx += offset
+	case io.SeekEnd:
+		if d.length+offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx = d.length + offset
+	}
+	return d.idx, nil
+}
+
+func TestDummyDataGenerator(t *testing.T) {
+	readAll := func(r io.Reader) string {
+		b, _ := ioutil.ReadAll(r)
+		return string(b)
+	}
+	checkEq := func(a, b string) {
+		if a != b {
+			t.Fatalf("Unexpected equality failure")
+		}
+	}
+
+	checkEq(readAll(NewDummyDataGen(0, 0)), "")
+
+	checkEq(readAll(NewDummyDataGen(10, 0)), readAll(NewDummyDataGen(10, int64(len(alphabets)))))
+
+	checkEq(readAll(NewDummyDataGen(100, 0)), readAll(NewDummyDataGen(50, 0))+readAll(NewDummyDataGen(50, 50)))
+
+	r := NewDummyDataGen(100, 0)
+	r.Seek(int64(len(alphabets)), 0)
+	checkEq(readAll(r), readAll(NewDummyDataGen(100-int64(len(alphabets)), 0)))
+}
+
+// Compares all the bytes returned by the given readers. Any Read
+// errors cause a `false` result. A string describing the error is
+// also returned.
+func cmpReaders(r1, r2 io.Reader) (bool, string) {
+	bufLen := 32 * 1024
+	b1, b2 := make([]byte, bufLen), make([]byte, bufLen)
+	for i := 0; true; i++ {
+		n1, e1 := io.ReadFull(r1, b1)
+		n2, e2 := io.ReadFull(r2, b2)
+		if n1 != n2 {
+			return false, fmt.Sprintf("Read %d != %d bytes from the readers", n1, n2)
+		}
+		if !bytes.Equal(b1[:n1], b2[:n2]) {
+			return false, fmt.Sprintf("After reading %d equal buffers (32Kib each), we got the following two strings:\n%v\n%v\n",
+				i, b1, b2)
+		}
+		// Check if stream has ended
+		if (e1 == io.ErrUnexpectedEOF && e2 == io.ErrUnexpectedEOF) || (e1 == io.EOF && e2 == io.EOF) {
+			break
+		}
+		if e1 != nil || e2 != nil {
+			return false, fmt.Sprintf("Got unexpected error values: %v == %v", e1, e2)
+		}
+	}
+	return true, ""
+}
+
+func TestCmpReaders(t *testing.T) {
+	{
+		r1 := bytes.NewBuffer([]byte("abc"))
+		r2 := bytes.NewBuffer([]byte("abc"))
+		ok, msg := cmpReaders(r1, r2)
+		if !(ok && msg == "") {
+			t.Fatalf("unexpected")
+		}
+	}
+
+	{
+		r1 := bytes.NewBuffer([]byte("abc"))
+		r2 := bytes.NewBuffer([]byte("abcd"))
+		ok, _ := cmpReaders(r1, r2)
+		if ok {
+			t.Fatalf("unexpected")
+		}
+	}
+}

cmd/dummy-object-layer_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"io"
+	"net/http"
 
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
@@ -59,19 +60,23 @@ func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix,
 	return
 }
 
-func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+func (api *DummyObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lock LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	return
 }
 
-func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	return
+}
+
+func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
@@ -83,15 +88,15 @@ func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, p
 	return
 }
 
-func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error) {
+func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
 	return
 }
 
@@ -154,3 +159,7 @@ func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
 func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
 	return
 }
+
+func (api *DummyObjectLayer) IsCompressionSupported() (b bool) {
+	return
+}

cmd/encryption-v1.go

@@ -37,12 +37,13 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errInsecureSSERequest   = errors.New("SSE-C requests require TLS connections")
 	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
 	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
 	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
 	// Additional Minio errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
+	// error returned when invalid encryption parameters are specified
+	errInvalidEncryptionParameters = errors.New("The encryption parameters are not applicable to this object")
 )
 
 const (
@@ -78,20 +79,35 @@ func hasServerSideEncryptionHeader(header http.Header) bool {
 	return crypto.S3.IsRequested(header) || crypto.SSEC.IsRequested(header)
 }
 
+// isEncryptedMultipart returns true if the current object is
+// uploaded by the user using multipart mechanism:
+// initiate new multipart, upload part, complete upload
+func isEncryptedMultipart(objInfo ObjectInfo) bool {
+	if len(objInfo.Parts) == 0 {
+		return false
+	}
+	if !crypto.IsMultiPart(objInfo.UserDefined) {
+		return false
+	}
+	for _, part := range objInfo.Parts {
+		_, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return false
+		}
+	}
+	// Further check if this object is uploaded using multipart mechanism
+	// by the user and it is not about XL internally splitting the
+	// object into parts in PutObject()
+	return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
+}
+
 // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
 // It returns the client provided key on success.
-func ParseSSECopyCustomerRequest(r *http.Request, metadata map[string]string) (key []byte, err error) {
-	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
-		// we cannot use r.TLS == nil here because Go's http implementation reflects on
-		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
-		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
-		// will always fail -> r.TLS is always nil even for TLS requests.
-		return nil, errInsecureSSERequest
-	}
-	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(r.Header) {
+func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {
+	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {
 		return nil, crypto.ErrIncompatibleEncryptionMethod
 	}
-	k, err := crypto.SSECopy.ParseHTTP(r.Header)
+	k, err := crypto.SSECopy.ParseHTTP(h)
 	return k[:], err
 }
 
@@ -104,13 +120,6 @@ func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
 // ParseSSECustomerHeader parses the SSE-C header fields and returns
 // the client provided key on success.
 func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
-	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
-		// we cannot use r.TLS == nil here because Go's http implementation reflects on
-		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
-		// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
-		// will always fail -> r.TLS is always nil even for TLS requests.
-		return nil, errInsecureSSERequest
-	}
 	if crypto.S3.IsRequested(header) && crypto.SSEC.IsRequested(header) {
 		return key, crypto.ErrIncompatibleEncryptionMethod
 	}
@@ -121,8 +130,6 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
 
 // This function rotates old to new key.
 func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map[string]string) error {
-	delete(metadata, crypto.SSECKey) // make sure we do not save the key by accident
-
 	switch {
 	default:
 		return errObjectTampered
@@ -147,13 +154,36 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 		}
 		copy(extKey[:], newKey)
 		sealedKey = objectKey.Seal(extKey, sealedKey.IV, crypto.SSEC.String(), bucket, object)
+		crypto.SSEC.CreateMetadata(metadata, sealedKey)
+		return nil
+	case crypto.S3.IsEncrypted(metadata):
+		if globalKMS == nil {
+			return errKMSNotConfigured
+		}
+		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
+		if err != nil {
+			return err
+		}
+		oldKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
+		if err != nil {
+			return err
+		}
+		var objectKey crypto.ObjectKey
+		if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
+			return err
+		}
+
+		newKey, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
+		if err != nil {
+			return err
+		}
+		sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
+		crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
 		return nil
 	}
 }
 
 func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) ([]byte, error) {
-	delete(metadata, crypto.SSECKey) // make sure we do not save the key by accident
-
 	var sealedKey crypto.SealedKey
 	if sseS3 {
 		if globalKMS == nil {
@@ -237,12 +267,11 @@ func DecryptCopyRequest(client io.Writer, r *http.Request, bucket, object string
 		err error
 	)
 	if crypto.SSECopy.IsRequested(r.Header) {
-		key, err = ParseSSECopyCustomerRequest(r, metadata)
+		key, err = ParseSSECopyCustomerRequest(r.Header, metadata)
 		if err != nil {
 			return nil, err
 		}
 	}
-	delete(metadata, crypto.SSECopyKey) // make sure we do not save the key by accident
 	return newDecryptWriter(client, key, bucket, object, 0, metadata)
 }
 
@@ -309,6 +338,114 @@ func newDecryptWriterWithObjectKey(client io.Writer, objectEncryptionKey []byte,
 	return writer, nil
 }
 
+// Adding support for reader based interface
+
+// DecryptRequestWithSequenceNumberR - same as
+// DecryptRequestWithSequenceNumber but with a reader
+func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	if crypto.S3.IsEncrypted(metadata) {
+		return newDecryptReader(client, nil, bucket, object, seqNumber, metadata)
+	}
+
+	key, err := ParseSSECustomerHeader(h)
+	if err != nil {
+		return nil, err
+	}
+	return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
+}
+
+// DecryptCopyRequestR - same as DecryptCopyRequest, but with a
+// Reader
+func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	var (
+		key []byte
+		err error
+	)
+	if crypto.SSECopy.IsRequested(h) {
+		key, err = ParseSSECopyCustomerRequest(h, metadata)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
+}
+
+func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
+	if err != nil {
+		return nil, err
+	}
+	return newDecryptReaderWithObjectKey(client, objectEncryptionKey, seqNumber, metadata)
+}
+
+func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	reader, err := sio.DecryptReader(client, sio.Config{
+		Key:            objectEncryptionKey,
+		SequenceNumber: seqNumber,
+	})
+	if err != nil {
+		return nil, crypto.ErrInvalidCustomerKey
+	}
+	return reader, nil
+}
+
+// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
+// reader
+func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
+	length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
+	io.Reader, error) {
+
+	bucket, object := oi.Bucket, oi.Name
+
+	// Single part case
+	if !isEncryptedMultipart(oi) {
+		var reader io.Reader
+		var err error
+		if copySource {
+			reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
+		} else {
+			reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
+		}
+		if err != nil {
+			return nil, err
+		}
+		return reader, nil
+	}
+
+	partDecRelOffset := int64(seqNumber) * sseDAREPackageBlockSize
+	partEncRelOffset := int64(seqNumber) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+
+	w := &DecryptBlocksReader{
+		reader:            inputReader,
+		startSeqNum:       seqNumber,
+		partDecRelOffset:  partDecRelOffset,
+		partEncRelOffset:  partEncRelOffset,
+		parts:             oi.Parts,
+		partIndex:         partStart,
+		header:            h,
+		bucket:            bucket,
+		object:            object,
+		customerKeyHeader: h.Get(crypto.SSECKey),
+		copySource:        copySource,
+	}
+
+	w.metadata = map[string]string{}
+	// Copy encryption metadata for internal use.
+	for k, v := range oi.UserDefined {
+		w.metadata[k] = v
+	}
+
+	if w.copySource {
+		w.customerKeyHeader = h.Get(crypto.SSECopyKey)
+	}
+
+	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
+		return nil, err
+	}
+
+	return w, nil
+}
+
 // DecryptRequestWithSequenceNumber decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
 func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
@@ -320,7 +457,6 @@ func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket,
 	if err != nil {
 		return nil, err
 	}
-	delete(metadata, crypto.SSECKey) // make sure we do not save the key by accident
 	return newDecryptWriter(client, key, bucket, object, seqNumber, metadata)
 }
 
@@ -330,7 +466,124 @@ func DecryptRequest(client io.Writer, r *http.Request, bucket, object string, me
 	return DecryptRequestWithSequenceNumber(client, r, bucket, object, 0, metadata)
 }
 
-// DecryptBlocksWriter - decrypts multipart parts, while implementing a io.Writer compatible interface.
+// DecryptBlocksReader - decrypts multipart parts, while implementing
+// a io.Reader compatible interface.
+type DecryptBlocksReader struct {
+	// Source of the encrypted content that will be decrypted
+	reader io.Reader
+	// Current decrypter for the current encrypted data block
+	decrypter io.Reader
+	// Start sequence number
+	startSeqNum uint32
+	// Current part index
+	partIndex int
+	// Parts information
+	parts          []objectPartInfo
+	header         http.Header
+	bucket, object string
+	metadata       map[string]string
+
+	partDecRelOffset, partEncRelOffset int64
+
+	copySource bool
+	// Customer Key
+	customerKeyHeader string
+}
+
+func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
+	m := make(map[string]string)
+	for k, v := range d.metadata {
+		m[k] = v
+	}
+	// Initialize the first decrypter; new decrypters will be
+	// initialized in Read() operation as needed.
+	var key []byte
+	var err error
+	if d.copySource {
+		if crypto.SSEC.IsEncrypted(d.metadata) {
+			d.header.Set(crypto.SSECopyKey, d.customerKeyHeader)
+			key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
+		}
+	} else {
+		if crypto.SSEC.IsEncrypted(d.metadata) {
+			d.header.Set(crypto.SSECKey, d.customerKeyHeader)
+			key, err = ParseSSECustomerHeader(d.header)
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	objectEncryptionKey, err := decryptObjectInfo(key, d.bucket, d.object, m)
+	if err != nil {
+		return err
+	}
+
+	var partIDbin [4]byte
+	binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
+
+	mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
+	mac.Write(partIDbin[:])
+	partEncryptionKey := mac.Sum(nil)
+
+	// Limit the reader, so the decryptor doesnt receive bytes
+	// from the next part (different DARE stream)
+	encLenToRead := d.parts[d.partIndex].Size - d.partEncRelOffset
+	decrypter, err := newDecryptReaderWithObjectKey(io.LimitReader(d.reader, encLenToRead), partEncryptionKey, d.startSeqNum, m)
+	if err != nil {
+		return err
+	}
+
+	d.decrypter = decrypter
+	return nil
+}
+
+func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
+	var err error
+	var n1 int
+	decPartSize, _ := sio.DecryptedSize(uint64(d.parts[d.partIndex].Size))
+	unreadPartLen := int64(decPartSize) - d.partDecRelOffset
+	if int64(len(p)) < unreadPartLen {
+		n1, err = d.decrypter.Read(p)
+		if err != nil {
+			return 0, err
+		}
+		d.partDecRelOffset += int64(n1)
+	} else {
+		n1, err = io.ReadFull(d.decrypter, p[:unreadPartLen])
+		if err != nil {
+			return 0, err
+		}
+
+		// We should now proceed to next part, reset all
+		// values appropriately.
+		d.partEncRelOffset = 0
+		d.partDecRelOffset = 0
+		d.startSeqNum = 0
+
+		d.partIndex++
+		if d.partIndex == len(d.parts) {
+			return n1, io.EOF
+		}
+
+		err = d.buildDecrypter(d.parts[d.partIndex].Number)
+		if err != nil {
+			return 0, err
+		}
+
+		n1, err = d.decrypter.Read(p[n1:])
+		if err != nil {
+			return 0, err
+		}
+
+		d.partDecRelOffset += int64(n1)
+	}
+
+	return len(p), nil
+}
+
+// DecryptBlocksWriter - decrypts  multipart parts, while implementing
+// a io.Writer compatible interface.
 type DecryptBlocksWriter struct {
 	// Original writer where the plain data will be written
 	writer io.Writer
@@ -364,7 +617,7 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 	if w.copySource {
 		if crypto.SSEC.IsEncrypted(w.metadata) {
 			w.req.Header.Set(crypto.SSECopyKey, w.customerKeyHeader)
-			key, err = ParseSSECopyCustomerRequest(w.req, w.metadata)
+			key, err = ParseSSECopyCustomerRequest(w.req.Header, w.metadata)
 		}
 	} else {
 		if crypto.SSEC.IsEncrypted(w.metadata) {
@@ -388,13 +641,6 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 	mac.Write(partIDbin[:])
 	partEncryptionKey := mac.Sum(nil)
 
-	// make sure we do not save the key by accident
-	if w.copySource {
-		delete(m, crypto.SSECopyKey)
-	} else {
-		delete(m, crypto.SSECKey)
-	}
-
 	// make sure to provide a NopCloser such that a Close
 	// on sio.decryptWriter doesn't close the underlying writer's
 	// close which perhaps can close the stream prematurely.
@@ -482,7 +728,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
 	var seqNumber uint32
 	var encStartOffset, encLength int64
 
-	if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
+	if !isEncryptedMultipart(objInfo) {
 		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
 
 		var writer io.WriteCloser
@@ -644,7 +890,7 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	if !crypto.IsEncrypted(o.UserDefined) {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
-	if len(o.Parts) == 0 || !crypto.IsMultiPart(o.UserDefined) {
+	if !isEncryptedMultipart(*o) {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
 			err = errObjectTampered // assign correct error type
@@ -663,6 +909,119 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	return size, nil
 }
 
+// GetDecryptedRange - To decrypt the range (off, length) of the
+// decrypted object stream, we need to read the range (encOff,
+// encLength) of the encrypted object stream to decrypt it, and
+// compute skipLen, the number of bytes to skip in the beginning of
+// the encrypted range.
+//
+// In addition we also compute the object part number for where the
+// requested range starts, along with the DARE sequence number within
+// that part. For single part objects, the partStart will be 0.
+func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, skipLen int64, seqNumber uint32, partStart int, err error) {
+	if !crypto.IsEncrypted(o.UserDefined) {
+		err = errors.New("Object is not encrypted")
+		return
+	}
+
+	if rs == nil {
+		// No range, so offsets refer to the whole object.
+		return 0, int64(o.Size), 0, 0, 0, nil
+	}
+
+	// Assemble slice of (decrypted) part sizes in `sizes`
+	var decObjSize int64 // decrypted total object size
+	var partSize uint64
+	partSize, err = sio.DecryptedSize(uint64(o.Size))
+	if err != nil {
+		return
+	}
+	sizes := []int64{int64(partSize)}
+	decObjSize = sizes[0]
+	if isEncryptedMultipart(*o) {
+		sizes = make([]int64, len(o.Parts))
+		decObjSize = 0
+		for i, part := range o.Parts {
+			partSize, err = sio.DecryptedSize(uint64(part.Size))
+			if err != nil {
+				return
+			}
+			t := int64(partSize)
+			sizes[i] = t
+			decObjSize += t
+		}
+	}
+
+	var off, length int64
+	off, length, err = rs.GetOffsetLength(decObjSize)
+	if err != nil {
+		return
+	}
+
+	// At this point, we have:
+	//
+	// 1. the decrypted part sizes in `sizes` (single element for
+	//    single part object) and total decrypted object size `decObjSize`
+	//
+	// 2. the (decrypted) start offset `off` and (decrypted)
+	//    length to read `length`
+	//
+	// These are the inputs to the rest of the algorithm below.
+
+	// Locate the part containing the start of the required range
+	var partEnd int
+	var cumulativeSum, encCumulativeSum int64
+	for i, size := range sizes {
+		if off < cumulativeSum+size {
+			partStart = i
+			break
+		}
+		cumulativeSum += size
+		encPartSize, _ := sio.EncryptedSize(uint64(size))
+		encCumulativeSum += int64(encPartSize)
+	}
+	// partStart is always found in the loop above,
+	// because off is validated.
+
+	sseDAREEncPackageBlockSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+	startPkgNum := (off - cumulativeSum) / sseDAREPackageBlockSize
+
+	// Now we can calculate the number of bytes to skip
+	skipLen = (off - cumulativeSum) % sseDAREPackageBlockSize
+
+	encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize
+	// Locate the part containing the end of the required range
+	endOffset := off + length - 1
+	for i1, size := range sizes[partStart:] {
+		i := partStart + i1
+		if endOffset < cumulativeSum+size {
+			partEnd = i
+			break
+		}
+		cumulativeSum += size
+		encPartSize, _ := sio.EncryptedSize(uint64(size))
+		encCumulativeSum += int64(encPartSize)
+	}
+	// partEnd is always found in the loop above, because off and
+	// length are validated.
+	endPkgNum := (endOffset - cumulativeSum) / sseDAREPackageBlockSize
+	// Compute endEncOffset with one additional DARE package (so
+	// we read the package containing the last desired byte).
+	endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize
+	// Check if the DARE package containing the end offset is a
+	// full sized package (as the last package in the part may be
+	// smaller)
+	lastPartSize, _ := sio.EncryptedSize(uint64(sizes[partEnd]))
+	if endEncOffset > encCumulativeSum+int64(lastPartSize) {
+		endEncOffset = encCumulativeSum + int64(lastPartSize)
+	}
+	encLength = endEncOffset - encOff
+	// Set the sequence number as the starting package number of
+	// the requested block
+	seqNumber = uint32(startPkgNum)
+	return encOff, encLength, skipLen, seqNumber, partStart, nil
+}
+
 // EncryptedSize returns the size of the object after encryption.
 // An encrypted object is always larger than a plain object
 // except for zero size objects.
@@ -713,28 +1072,25 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
 // decryption succeeded.
 //
 // DecryptObjectInfo also returns whether the object is encrypted or not.
-func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
+func DecryptObjectInfo(info ObjectInfo, headers http.Header) (encrypted bool, err error) {
 	// Directories are never encrypted.
 	if info.IsDir {
-		return ErrNone, false
+		return false, nil
 	}
 	// disallow X-Amz-Server-Side-Encryption header on HEAD and GET
 	if crypto.S3.IsRequested(headers) {
-		apiErr = ErrInvalidEncryptionParameters
+		err = errInvalidEncryptionParameters
 		return
 	}
-	if apiErr, encrypted = ErrNone, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSEC.IsRequested(headers) {
-		apiErr = ErrInvalidEncryptionParameters
+	if err, encrypted = nil, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSEC.IsRequested(headers) {
+		err = errInvalidEncryptionParameters
 	} else if encrypted {
 		if (crypto.SSEC.IsEncrypted(info.UserDefined) && !crypto.SSEC.IsRequested(headers)) ||
 			(crypto.S3.IsEncrypted(info.UserDefined) && crypto.SSEC.IsRequested(headers)) {
-			apiErr = ErrSSEEncryptedObject
+			err = errEncryptedObject
 			return
 		}
-		var err error
-		if info.Size, err = info.DecryptedSize(); err != nil {
-			apiErr = toAPIErrorCode(err)
-		}
+		_, err = info.DecryptedSize()
 	}
 	return
 }

cmd/encryption-v1_test.go

@@ -18,11 +18,12 @@ package cmd
 
 import (
 	"bytes"
-	"encoding/base64"
 	"net/http"
 	"testing"
 
+	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/sio"
 )
 
 var hasServerSideEncryptionHeaderTests = []struct {
@@ -105,237 +106,6 @@ func TesthasSSECustomerHeader(t *testing.T) {
 	}
 }
 
-var parseSSECustomerRequestTests = []struct {
-	headers map[string]string
-	useTLS  bool
-	err     error
-}{
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: nil,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: false, err: errInsecureSSERequest,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES 256",
-			crypto.SSECKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
-			crypto.SSECKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerAlgorithm,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
-			crypto.SSECKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
-			crypto.SSECKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
-			crypto.SSECKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
-		},
-		useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
-			crypto.SSECKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "", // 7
-			crypto.SSECKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		useTLS: true, err: crypto.ErrMissingCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECKeyMD5:    "",
-		},
-		useTLS: true, err: crypto.ErrMissingCustomerKeyMD5,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECAlgorithm: "AES256",
-			crypto.SSECKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECKeyMD5:    "",
-			crypto.SSEHeader:     "",
-		},
-		useTLS: true, err: crypto.ErrIncompatibleEncryptionMethod,
-	},
-}
-
-func TestParseSSECustomerRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	for i, test := range parseSSECustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		request := &http.Request{}
-		request.Header = headers
-		globalIsSSL = test.useTLS
-
-		_, err := ParseSSECustomerRequest(request)
-		if err != test.err {
-			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
-		}
-		key := request.Header.Get(crypto.SSECKey)
-		if (err == nil || err == crypto.ErrCustomerKeyMD5Mismatch) && key != "" {
-			t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
-		}
-
-	}
-}
-
-var parseSSECopyCustomerRequestTests = []struct {
-	headers  map[string]string
-	metadata map[string]string
-	useTLS   bool
-	err      error
-}{
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: nil,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": base64.StdEncoding.EncodeToString(make([]byte, 64))},
-		useTLS:   true, err: crypto.ErrIncompatibleEncryptionMethod,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   false, err: errInsecureSSERequest,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES 256",
-			crypto.SSECopyKey:       "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
-			crypto.SSECopyKeyMD5:    "bY4wkxQejw9mUJfo72k53A==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerAlgorithm,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
-			crypto.SSECopyKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
-			crypto.SSECopyKeyMD5:    "H+jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
-			crypto.SSECopyKeyMD5:    " +jq/LwEOEO90YtiTuNFVw==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrCustomerKeyMD5Mismatch,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
-			crypto.SSECopyKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrInvalidCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "", // 7
-			crypto.SSECopyKeyMD5:    "9KPgDdZNTHimuYCwnJTp5g==",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrMissingCustomerKey,
-	},
-	{
-		headers: map[string]string{
-			crypto.SSECopyAlgorithm: "AES256",
-			crypto.SSECopyKey:       "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
-			crypto.SSECopyKeyMD5:    "",
-		},
-		metadata: map[string]string{},
-		useTLS:   true, err: crypto.ErrMissingCustomerKeyMD5,
-	},
-}
-
-func TestParseSSECopyCustomerRequest(t *testing.T) {
-	defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
-	for i, test := range parseSSECopyCustomerRequestTests {
-		headers := http.Header{}
-		for k, v := range test.headers {
-			headers.Set(k, v)
-		}
-		request := &http.Request{}
-		request.Header = headers
-		globalIsSSL = test.useTLS
-
-		_, err := ParseSSECopyCustomerRequest(request, test.metadata)
-		if err != test.err {
-			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
-		}
-		key := request.Header.Get(crypto.SSECopyKey)
-		if (err == nil || err == crypto.ErrCustomerKeyMD5Mismatch) && key != "" {
-			t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
-		}
-	}
-}
-
 var encryptRequestTests = []struct {
 	header   map[string]string
 	metadata map[string]string
@@ -374,9 +144,6 @@ func TestEncryptRequest(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
-		if key, ok := test.metadata[crypto.SSECKey]; ok {
-			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
-		}
 		if kdf, ok := test.metadata[crypto.SSESealAlgorithm]; !ok {
 			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
 		}
@@ -521,43 +288,43 @@ func TestDecryptRequest(t *testing.T) {
 var decryptObjectInfoTests = []struct {
 	info    ObjectInfo
 	headers http.Header
-	expErr  APIErrorCode
+	expErr  error
 }{
 	{
 		info:    ObjectInfo{Size: 100},
 		headers: http.Header{},
-		expErr:  ErrNone,
+		expErr:  nil,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrNone,
+		expErr:  nil,
 	},
 	{
 		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrNone,
+		expErr:  nil,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSECSealedKey: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
 		headers: http.Header{},
-		expErr:  ErrSSEEncryptedObject,
+		expErr:  errEncryptedObject,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrInvalidEncryptionParameters,
+		expErr:  errInvalidEncryptionParameters,
 	},
 	{
 		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrObjectTampered,
+		expErr:  errObjectTampered,
 	},
 }
 
 func TestDecryptObjectInfo(t *testing.T) {
 	for i, test := range decryptObjectInfoTests {
-		if err, encrypted := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
+		if encrypted, err := DecryptObjectInfo(test.info, test.headers); err != test.expErr {
 			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
 		} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
 			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
@@ -566,3 +333,285 @@ func TestDecryptObjectInfo(t *testing.T) {
 		}
 	}
 }
+
+func TestGetDecryptedRange(t *testing.T) {
+	var (
+		pkgSz     = int64(64) * humanize.KiByte
+		minPartSz = int64(5) * humanize.MiByte
+		maxPartSz = int64(5) * humanize.GiByte
+
+		getEncSize = func(s int64) int64 {
+			v, _ := sio.EncryptedSize(uint64(s))
+			return int64(v)
+		}
+		udMap = func(isMulti bool) map[string]string {
+			m := map[string]string{
+				crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+				crypto.SSEMultipart:     "1",
+			}
+			if !isMulti {
+				delete(m, crypto.SSEMultipart)
+			}
+			return m
+		}
+	)
+
+	// Single part object tests
+	var (
+		mkSPObj = func(s int64) ObjectInfo {
+			return ObjectInfo{
+				Size:        getEncSize(s),
+				UserDefined: udMap(false),
+			}
+		}
+	)
+
+	testSP := []struct {
+		decSz int64
+		oi    ObjectInfo
+	}{
+		{0, mkSPObj(0)},
+		{1, mkSPObj(1)},
+		{pkgSz - 1, mkSPObj(pkgSz - 1)},
+		{pkgSz, mkSPObj(pkgSz)},
+		{2*pkgSz - 1, mkSPObj(2*pkgSz - 1)},
+		{minPartSz, mkSPObj(minPartSz)},
+		{maxPartSz, mkSPObj(maxPartSz)},
+	}
+
+	for i, test := range testSP {
+		{
+			// nil range
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != getEncSize(test.decSz) {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		if test.decSz >= 10 {
+			// first 10 bytes
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 0, 9})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			var rLen = pkgSz + 32
+			if test.decSz < pkgSz {
+				rLen = test.decSz + 32
+			}
+			if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != rLen {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		kb32 := int64(32) * humanize.KiByte
+		if test.decSz >= (64+32)*humanize.KiByte {
+			// Skip the first 32Kib, and read the next 64Kib
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, kb32, 3*kb32 - 1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			var rLen = (pkgSz + 32) * 2
+			if test.decSz < 2*pkgSz {
+				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32)
+			}
+			if skip != kb32 || sn != 0 || ps != 0 || o != 0 || l != rLen {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		if test.decSz >= (64*2+32)*humanize.KiByte {
+			// Skip the first 96Kib and read the next 64Kib
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 3 * kb32, 5*kb32 - 1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			var rLen = (pkgSz + 32) * 2
+			if test.decSz-pkgSz < 2*pkgSz {
+				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32*2)
+			}
+			if skip != kb32 || sn != 1 || ps != 0 || o != pkgSz+32 || l != rLen {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+	}
+
+	// Multipart object tests
+	var (
+		// make a multipart object-info given part sizes
+		mkMPObj = func(sizes []int64) ObjectInfo {
+			r := make([]objectPartInfo, len(sizes))
+			sum := int64(0)
+			for i, s := range sizes {
+				r[i].Number = i
+				r[i].Size = int64(getEncSize(s))
+				sum += r[i].Size
+			}
+			return ObjectInfo{
+				Size:        sum,
+				UserDefined: udMap(true),
+				Parts:       r,
+			}
+		}
+		// Simple useful utilities
+		repeat = func(k int64, n int) []int64 {
+			a := []int64{}
+			for i := 0; i < n; i++ {
+				a = append(a, k)
+			}
+			return a
+		}
+		lsum = func(s []int64) int64 {
+			sum := int64(0)
+			for _, i := range s {
+				if i < 0 {
+					return -1
+				}
+				sum += i
+			}
+			return sum
+		}
+		esum = func(oi ObjectInfo) int64 {
+			sum := int64(0)
+			for _, i := range oi.Parts {
+				sum += i.Size
+			}
+			return sum
+		}
+	)
+
+	s1 := []int64{5487701, 5487799, 3}
+	s2 := repeat(5487701, 5)
+	s3 := repeat(maxPartSz, 10000)
+	testMPs := []struct {
+		decSizes []int64
+		oi       ObjectInfo
+	}{
+		{s1, mkMPObj(s1)},
+		{s2, mkMPObj(s2)},
+		{s3, mkMPObj(s3)},
+	}
+
+	// This function is a reference (re-)implementation of
+	// decrypted range computation, written solely for the purpose
+	// of the unit tests.
+	//
+	// `s` gives the decrypted part sizes, and the other
+	// parameters describe the desired read segment. When
+	// `isFromEnd` is true, `skipLen` argument is ignored.
+	decryptedRangeRef := func(s []int64, skipLen, readLen int64, isFromEnd bool) (o, l, skip int64, sn uint32, ps int) {
+		oSize := lsum(s)
+		if isFromEnd {
+			skipLen = oSize - readLen
+		}
+		if skipLen < 0 || readLen < 0 || oSize < 0 || skipLen+readLen > oSize {
+			t.Fatalf("Impossible read specified: %d %d %d", skipLen, readLen, oSize)
+		}
+
+		var cumulativeSum, cumulativeEncSum int64
+		toRead := readLen
+		readStart := false
+		for i, v := range s {
+			partOffset := int64(0)
+			partDarePkgOffset := int64(0)
+			if !readStart && cumulativeSum+v > skipLen {
+				// Read starts at the current part
+				readStart = true
+
+				partOffset = skipLen - cumulativeSum
+
+				// All return values except `l` are
+				// calculated here.
+				sn = uint32(partOffset / pkgSz)
+				skip = partOffset % pkgSz
+				ps = i
+				o = cumulativeEncSum + int64(sn)*(pkgSz+32)
+
+				partDarePkgOffset = partOffset - skip
+			}
+			if readStart {
+				currentPartBytes := v - partOffset
+				currentPartDareBytes := v - partDarePkgOffset
+				if currentPartBytes < toRead {
+					toRead -= currentPartBytes
+					l += getEncSize(currentPartDareBytes)
+				} else {
+					// current part has the last
+					// byte required
+					lbPartOffset := partOffset + toRead - 1
+
+					// round up the lbPartOffset
+					// to the end of the
+					// corresponding DARE package
+					lbPkgEndOffset := lbPartOffset - (lbPartOffset % pkgSz) + pkgSz
+					if lbPkgEndOffset > v {
+						lbPkgEndOffset = v
+					}
+					bytesToDrop := v - lbPkgEndOffset
+
+					// Last segment to update `l`
+					l += getEncSize(currentPartDareBytes - bytesToDrop)
+					break
+				}
+			}
+
+			cumulativeSum += v
+			cumulativeEncSum += getEncSize(v)
+		}
+		return
+	}
+
+	for i, test := range testMPs {
+		{
+			// nil range
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			if o != 0 || l != esum(test.oi) || skip != 0 || sn != 0 || ps != 0 {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		// Skip 1Mib and read 1Mib (in the decrypted object)
+		//
+		// The check below ensures the object is large enough
+		// for the read.
+		if lsum(test.decSizes) >= 2*humanize.MiByte {
+			skipLen, readLen := int64(1)*humanize.MiByte, int64(1)*humanize.MiByte
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, skipLen, skipLen + readLen - 1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+
+			oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, skipLen, readLen, false)
+			if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
+					i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
+			}
+		}
+
+		// Read the last 6Mib+1 bytes of the (decrypted)
+		// object
+		//
+		// The check below ensures the object is large enough
+		// for the read.
+		readLen := int64(6)*humanize.MiByte + 1
+		if lsum(test.decSizes) >= readLen {
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{true, -readLen, -1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+
+			oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, 0, readLen, true)
+			if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
+					i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
+			}
+		}
+
+	}
+}

cmd/endpoint.go

@@ -168,6 +168,18 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 // EndpointList - list of same type of endpoint.
 type EndpointList []Endpoint
 
+// Nodes - returns number of unique servers.
+func (endpoints EndpointList) Nodes() int {
+	uniqueNodes := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if uniqueNodes.Contains(endpoint.Host) {
+			continue
+		}
+		uniqueNodes.Add(endpoint.Host)
+	}
+	return len(uniqueNodes)
+}
+
 // IsHTTPS - returns true if secure for URLEndpointType.
 func (endpoints EndpointList) IsHTTPS() bool {
 	return endpoints[0].IsHTTPS()

cmd/erasure-decode.go

@@ -127,8 +127,8 @@ func (p *parallelReader) Read() ([][]byte, error) {
 	return nil, errXLReadQuorum
 }
 
-// ReadFile reads from readers, reconstructs data if needed and writes the data to the writer.
-func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, readers []*bitrotReader, offset, length, totalLength int64) error {
+// Decode reads from readers, reconstructs data if needed and writes the data to the writer.
+func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []*bitrotReader, offset, length, totalLength int64) error {
 	if offset < 0 || length < 0 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
@@ -141,27 +141,27 @@ func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, readers
 		return nil
 	}
 
-	reader := newParallelReader(readers, s.dataBlocks, offset, totalLength, s.blockSize)
+	reader := newParallelReader(readers, e.dataBlocks, offset, totalLength, e.blockSize)
 
-	startBlock := offset / s.blockSize
-	endBlock := (offset + length) / s.blockSize
+	startBlock := offset / e.blockSize
+	endBlock := (offset + length) / e.blockSize
 
 	var bytesWritten int64
 	for block := startBlock; block <= endBlock; block++ {
 		var blockOffset, blockLength int64
 		switch {
 		case startBlock == endBlock:
-			blockOffset = offset % s.blockSize
+			blockOffset = offset % e.blockSize
 			blockLength = length
 		case block == startBlock:
-			blockOffset = offset % s.blockSize
-			blockLength = s.blockSize - blockOffset
+			blockOffset = offset % e.blockSize
+			blockLength = e.blockSize - blockOffset
 		case block == endBlock:
 			blockOffset = 0
-			blockLength = (offset + length) % s.blockSize
+			blockLength = (offset + length) % e.blockSize
 		default:
 			blockOffset = 0
-			blockLength = s.blockSize
+			blockLength = e.blockSize
 		}
 		if blockLength == 0 {
 			break
@@ -170,11 +170,11 @@ func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, readers
 		if err != nil {
 			return err
 		}
-		if err = s.ErasureDecodeDataBlocks(bufs); err != nil {
+		if err = e.DecodeDataBlocks(bufs); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
-		n, err := writeDataBlocks(ctx, writer, bufs, s.dataBlocks, blockOffset, blockLength)
+		n, err := writeDataBlocks(ctx, writer, bufs, e.dataBlocks, blockOffset, blockLength)
 		if err != nil {
 			return err
 		}

cmd/erasure-decode_test.go

@@ -32,7 +32,7 @@ func (d badDisk) ReadFile(volume string, path string, offset int64, buf []byte,
 	return 0, errFaultyDisk
 }
 
-var erasureReadFileTests = []struct {
+var erasureDecodeTests = []struct {
 	dataBlocks                   int
 	onDisks, offDisks            int
 	blocksize, data              int64
@@ -81,13 +81,13 @@ var erasureReadFileTests = []struct {
 	{dataBlocks: 8, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 11, length: int64(blockSizeV1) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},            // 37
 }
 
-func TestErasureReadFile(t *testing.T) {
-	for i, test := range erasureReadFileTests {
+func TestErasureDecode(t *testing.T) {
+	for i, test := range erasureDecodeTests {
 		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -108,7 +108,7 @@ func TestErasureReadFile(t *testing.T) {
 		for i, disk := range disks {
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", writeAlgorithm)
 		}
-		n, err := storage.CreateFile(context.Background(), bytes.NewReader(data[:]), writers, buffer, storage.dataBlocks+1)
+		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[:]), writers, buffer, erasure.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
@@ -129,12 +129,12 @@ func TestErasureReadFile(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, storage.dataBlocks)
+			endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, erasure.dataBlocks)
 			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", writeAlgorithm, endOffset, writers[index].Sum())
 		}
 
 		writer := bytes.NewBuffer(nil)
-		err = storage.ReadFile(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
+		err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -157,7 +157,7 @@ func TestErasureReadFile(t *testing.T) {
 				if disk == OfflineDisk {
 					continue
 				}
-				endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, storage.dataBlocks)
+				endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, erasure.dataBlocks)
 				bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", writeAlgorithm, endOffset, writers[index].Sum())
 			}
 			for j := range disks[:test.offDisks] {
@@ -167,7 +167,7 @@ func TestErasureReadFile(t *testing.T) {
 				bitrotReaders[0] = nil
 			}
 			writer.Reset()
-			err = storage.ReadFile(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
+			err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -184,10 +184,10 @@ func TestErasureReadFile(t *testing.T) {
 	}
 }
 
-// Test erasureReadFile with random offset and lengths.
+// Test erasureDecode with random offset and lengths.
 // This test is t.Skip()ed as it a long time to run, hence should be run
 // explicitly after commenting out t.Skip()
-func TestErasureReadFileRandomOffsetLength(t *testing.T) {
+func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 	// Comment the following line to run this test.
 	t.SkipNow()
 	// Initialize environment needed for the test.
@@ -201,7 +201,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 	defer setup.Remove()
 	disks := setup.disks
-	storage, err := NewErasureStorage(context.Background(), dataBlocks, parityBlocks, blockSize)
+	erasure, err := NewErasure(context.Background(), dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -226,7 +226,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 	// Create a test file to read from.
 	buffer := make([]byte, blockSize, 2*blockSize)
-	n, err := storage.CreateFile(context.Background(), bytes.NewReader(data), writers, buffer, storage.dataBlocks+1)
+	n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -239,7 +239,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 	buf := &bytes.Buffer{}
 
-	// Verify erasureReadFile() for random offsets and lengths.
+	// Verify erasure.Decode() for random offsets and lengths.
 	for i := 0; i < iterations; i++ {
 		offset := r.Int63n(length)
 		readLen := r.Int63n(length - offset)
@@ -252,10 +252,10 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(offset, readLen, length, blockSize, storage.dataBlocks)
+			endOffset := getErasureShardFileEndOffset(offset, readLen, length, blockSize, erasure.dataBlocks)
 			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", DefaultBitrotAlgorithm, endOffset, writers[index].Sum())
 		}
-		err = storage.ReadFile(context.Background(), buf, bitrotReaders, offset, readLen, length)
+		err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
 		}
@@ -269,14 +269,14 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 // Benchmarks
 
-func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
+func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
 	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
 	disks := setup.disks
-	storage, err := NewErasureStorage(context.Background(), data, parity, blockSizeV1)
+	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -291,7 +291,7 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 
 	content := make([]byte, size)
 	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
-	_, err = storage.CreateFile(context.Background(), bytes.NewReader(content), writers, buffer, storage.dataBlocks+1)
+	_, err = erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
 	if err != nil {
 		b.Fatalf("failed to create erasure test file: %v", err)
 	}
@@ -312,62 +312,62 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 			if writers[index] == nil {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(0, size, size, storage.blockSize, storage.dataBlocks)
+			endOffset := getErasureShardFileEndOffset(0, size, size, erasure.blockSize, erasure.dataBlocks)
 			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", DefaultBitrotAlgorithm, endOffset, writers[index].Sum())
 		}
-		if err = storage.ReadFile(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size); err != nil {
+		if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size); err != nil {
 			panic(err)
 		}
 	}
 }
 
-func BenchmarkErasureReadQuick(b *testing.B) {
+func BenchmarkErasureDecodeQuick(b *testing.B) {
 	const size = 12 * 1024 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
-	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 0, size, b) })
+	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 1, size, b) })
 }
 
-func BenchmarkErasureRead_4_64KB(b *testing.B) {
+func BenchmarkErasureDecode_4_64KB(b *testing.B) {
 	const size = 64 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
-	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
-	b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 2, size, b) })
-	b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 2, 0, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 0, size, b) })
+	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 1, size, b) })
+	b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 2, size, b) })
+	b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 2, 0, size, b) })
 }
 
-func BenchmarkErasureRead_8_20MB(b *testing.B) {
+func BenchmarkErasureDecode_8_20MB(b *testing.B) {
 	const size = 20 * 1024 * 1024
-	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 0, size, b) })
-	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 1, size, b) })
-	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 0, size, b) })
-	b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 1, size, b) })
-	b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 4, size, b) })
-	b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 2, 2, size, b) })
-	b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 4, 0, size, b) })
+	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 0, size, b) })
+	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 1, size, b) })
+	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 1, 0, size, b) })
+	b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 1, 1, size, b) })
+	b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 4, size, b) })
+	b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 2, 2, size, b) })
+	b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 4, 0, size, b) })
 }
 
-func BenchmarkErasureRead_12_30MB(b *testing.B) {
+func BenchmarkErasureDecode_12_30MB(b *testing.B) {
 	const size = 30 * 1024 * 1024
-	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 0, size, b) })
-	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 1, size, b) })
-	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 0, size, b) })
-	b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 1, size, b) })
-	b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 6, size, b) })
-	b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 3, 3, size, b) })
-	b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 6, 0, size, b) })
+	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 0, size, b) })
+	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 1, size, b) })
+	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 1, 0, size, b) })
+	b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 1, 1, size, b) })
+	b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 6, size, b) })
+	b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 3, 3, size, b) })
+	b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 6, 0, size, b) })
 }
 
-func BenchmarkErasureRead_16_40MB(b *testing.B) {
+func BenchmarkErasureDecode_16_40MB(b *testing.B) {
 	const size = 40 * 1024 * 1024
-	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 0, size, b) })
-	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 1, size, b) })
-	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 0, size, b) })
-	b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 1, size, b) })
-	b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 8, size, b) })
-	b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 4, 4, size, b) })
-	b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 8, 0, size, b) })
+	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 0, size, b) })
+	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 1, size, b) })
+	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 1, 0, size, b) })
+	b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 1, 1, size, b) })
+	b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 8, size, b) })
+	b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 4, 4, size, b) })
+	b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 8, 0, size, b) })
 }

cmd/erasure-encode.go

@@ -69,8 +69,8 @@ func (p *parallelWriter) Append(ctx context.Context, blocks [][]byte) error {
 	return reduceWriteQuorumErrs(ctx, p.errs, objectOpIgnoredErrs, p.writeQuorum)
 }
 
-// CreateFile reads from the reader, erasure-encodes the data and writes to the writers.
-func (s *ErasureStorage) CreateFile(ctx context.Context, src io.Reader, writers []*bitrotWriter, buf []byte, quorum int) (total int64, err error) {
+// Encode reads from the reader, erasure-encodes the data and writes to the writers.
+func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []*bitrotWriter, buf []byte, quorum int) (total int64, err error) {
 	writer := &parallelWriter{
 		writers:     writers,
 		writeQuorum: quorum,
@@ -90,7 +90,7 @@ func (s *ErasureStorage) CreateFile(ctx context.Context, src io.Reader, writers
 			break
 		}
 		// We take care of the situation where if n == 0 and total == 0 by creating empty data and parity files.
-		blocks, err = s.ErasureEncode(ctx, buf[:n])
+		blocks, err = e.EncodeData(ctx, buf[:n])
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return 0, err

cmd/erasure-encode_test.go

@@ -28,13 +28,17 @@ import (
 
 type badDisk struct{ StorageAPI }
 
+func (a badDisk) String() string {
+	return "bad-disk"
+}
+
 func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
 	return errFaultyDisk
 }
 
 const oneMiByte = 1 * humanize.MiByte
 
-var erasureCreateFileTests = []struct {
+var erasureEncodeTests = []struct {
 	dataBlocks                   int
 	onDisks, offDisks            int
 	blocksize, data              int64
@@ -64,14 +68,14 @@ var erasureCreateFileTests = []struct {
 	{dataBlocks: 10, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                // 19
 }
 
-func TestErasureCreateFile(t *testing.T) {
-	for i, test := range erasureCreateFileTests {
+func TestErasureEncode(t *testing.T) {
+	for i, test := range erasureEncodeTests {
 		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		disks := setup.disks
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -90,7 +94,7 @@ func TestErasureCreateFile(t *testing.T) {
 			}
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", test.algorithm)
 		}
-		n, err := storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, storage.dataBlocks+1)
+		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -119,7 +123,7 @@ func TestErasureCreateFile(t *testing.T) {
 			if test.offDisks > 0 {
 				writers[0] = nil
 			}
-			n, err = storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, storage.dataBlocks+1)
+			n, err = erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -138,13 +142,13 @@ func TestErasureCreateFile(t *testing.T) {
 
 // Benchmarks
 
-func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
+func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
 	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(context.Background(), data, parity, blockSizeV1)
+	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -170,50 +174,50 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
 			}
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", DefaultBitrotAlgorithm)
 		}
-		_, err := storage.CreateFile(context.Background(), bytes.NewReader(content), writers, buffer, storage.dataBlocks+1)
+		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
 		if err != nil {
 			panic(err)
 		}
 	}
 }
 
-func BenchmarkErasureWriteQuick(b *testing.B) {
+func BenchmarkErasureEncodeQuick(b *testing.B) {
 	const size = 12 * 1024 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 1, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_4_64KB(b *testing.B) {
+func BenchmarkErasureEncode_4_64KB(b *testing.B) {
 	const size = 64 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 1, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_8_20MB(b *testing.B) {
+func BenchmarkErasureEncode_8_20MB(b *testing.B) {
 	const size = 20 * 1024 * 1024
-	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 0, size, b) })
-	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 1, size, b) })
-	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 1, 0, size, b) })
-	b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 3, size, b) })
-	b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 3, 0, size, b) })
+	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 0, size, b) })
+	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 1, size, b) })
+	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 1, 0, size, b) })
+	b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 3, size, b) })
+	b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 3, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_12_30MB(b *testing.B) {
+func BenchmarkErasureEncode_12_30MB(b *testing.B) {
 	const size = 30 * 1024 * 1024
-	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 0, size, b) })
-	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 1, size, b) })
-	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 1, 0, size, b) })
-	b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 5, size, b) })
-	b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 5, 0, size, b) })
+	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 0, size, b) })
+	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 1, size, b) })
+	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 1, 0, size, b) })
+	b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 5, size, b) })
+	b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 5, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_16_40MB(b *testing.B) {
+func BenchmarkErasureEncode_16_40MB(b *testing.B) {
 	const size = 40 * 1024 * 1024
-	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 0, size, b) })
-	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 1, size, b) })
-	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 1, 0, size, b) })
-	b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 7, size, b) })
-	b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 7, 0, size, b) })
+	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 0, size, b) })
+	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 1, size, b) })
+	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 1, 0, size, b) })
+	b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 7, size, b) })
+	b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 7, 0, size, b) })
 }

cmd/erasure-heal.go

@@ -23,20 +23,20 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-// HealFile heals the shard files on non-nil writers. Note that the quorum passed is 1
+// Heal heals the shard files on non-nil writers. Note that the quorum passed is 1
 // as healing should continue even if it has been successful healing only one shard file.
-func (s ErasureStorage) HealFile(ctx context.Context, readers []*bitrotReader, writers []*bitrotWriter, size int64) error {
+func (e Erasure) Heal(ctx context.Context, readers []*bitrotReader, writers []*bitrotWriter, size int64) error {
 	r, w := io.Pipe()
 	go func() {
-		if err := s.ReadFile(ctx, w, readers, 0, size, size); err != nil {
+		if err := e.Decode(ctx, w, readers, 0, size, size); err != nil {
 			w.CloseWithError(err)
 			return
 		}
 		w.Close()
 	}()
-	buf := make([]byte, s.blockSize)
+	buf := make([]byte, e.blockSize)
 	// quorum is 1 because CreateFile should continue writing as long as we are writing to even 1 disk.
-	n, err := s.CreateFile(ctx, r, writers, buf, 1)
+	n, err := e.Encode(ctx, r, writers, buf, 1)
 	if err != nil {
 		return err
 	}

cmd/erasure-heal_test.go

@@ -24,7 +24,7 @@ import (
 	"testing"
 )
 
-var erasureHealFileTests = []struct {
+var erasureHealTests = []struct {
 	dataBlocks, disks int
 
 	// number of offline disks is also number of staleDisks for
@@ -60,8 +60,8 @@ var erasureHealFileTests = []struct {
 	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte * 64, algorithm: SHA256, shouldFail: false},              // 19
 }
 
-func TestErasureHealFile(t *testing.T) {
-	for i, test := range erasureHealFileTests {
+func TestErasureHeal(t *testing.T) {
+	for i, test := range erasureHealTests {
 		if test.offDisks < test.badStaleDisks {
 			// test case sanity check
 			t.Fatalf("Test %d: Bad test case - number of stale disks cannot be less than number of badstale disks", i)
@@ -73,7 +73,7 @@ func TestErasureHealFile(t *testing.T) {
 			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
 		}
 		disks := setup.disks
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -88,7 +88,7 @@ func TestErasureHealFile(t *testing.T) {
 		for i, disk := range disks {
 			writers[i] = newBitrotWriter(disk, "testbucket", "testobject", test.algorithm)
 		}
-		_, err = storage.CreateFile(context.Background(), bytes.NewReader(data), writers, buffer, storage.dataBlocks+1)
+		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
@@ -96,7 +96,7 @@ func TestErasureHealFile(t *testing.T) {
 
 		readers := make([]*bitrotReader, len(disks))
 		for i, disk := range disks {
-			shardFilesize := getErasureShardFileSize(test.blocksize, test.size, storage.dataBlocks)
+			shardFilesize := getErasureShardFileSize(test.blocksize, test.size, erasure.dataBlocks)
 			readers[i] = newBitrotReader(disk, "testbucket", "testobject", test.algorithm, shardFilesize, writers[i].Sum())
 		}
 
@@ -126,7 +126,7 @@ func TestErasureHealFile(t *testing.T) {
 		}
 
 		// test case setup is complete - now call Healfile()
-		err = storage.HealFile(context.Background(), readers, staleWriters, test.size)
+		err = erasure.Heal(context.Background(), readers, staleWriters, test.size)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but it failed with: %v", i, err)
 		}

cmd/erasure-utils.go

@@ -81,7 +81,9 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		if write < int64(len(block)) {
 			n, err := io.Copy(dst, bytes.NewReader(block[:write]))
 			if err != nil {
-				logger.LogIf(ctx, err)
+				if err != io.ErrClosedPipe {
+					logger.LogIf(ctx, err)
+				}
 				return 0, err
 			}
 			totalWritten += n
@@ -90,7 +92,10 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		// Copy the block.
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
-			logger.LogIf(ctx, err)
+			// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
+			if err != io.ErrClosedPipe {
+				logger.LogIf(ctx, err)
+			}
 			return 0, err
 		}
 

cmd/erasure.go

@@ -23,26 +23,23 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-// OfflineDisk represents an unavailable disk.
-var OfflineDisk StorageAPI // zero value is nil
-
-// ErasureStorage - erasure encoding details.
-type ErasureStorage struct {
-	erasure                  reedsolomon.Encoder
+// Erasure - erasure encoding details.
+type Erasure struct {
+	encoder                  reedsolomon.Encoder
 	dataBlocks, parityBlocks int
 	blockSize                int64
 }
 
-// NewErasureStorage creates a new ErasureStorage.
-func NewErasureStorage(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
+// NewErasure creates a new ErasureStorage.
+func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (e Erasure, err error) {
 	shardsize := int(ceilFrac(blockSize, int64(dataBlocks)))
 	erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize))
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return s, err
+		return e, err
 	}
-	s = ErasureStorage{
-		erasure:      erasure,
+	e = Erasure{
+		encoder:      erasure,
 		dataBlocks:   dataBlocks,
 		parityBlocks: parityBlocks,
 		blockSize:    blockSize,
@@ -50,30 +47,30 @@ func NewErasureStorage(ctx context.Context, dataBlocks, parityBlocks int, blockS
 	return
 }
 
-// ErasureEncode encodes the given data and returns the erasure-coded data.
+// EncodeData encodes the given data and returns the erasure-coded data.
 // It returns an error if the erasure coding failed.
-func (s *ErasureStorage) ErasureEncode(ctx context.Context, data []byte) ([][]byte, error) {
+func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error) {
 	if len(data) == 0 {
-		return make([][]byte, s.dataBlocks+s.parityBlocks), nil
+		return make([][]byte, e.dataBlocks+e.parityBlocks), nil
 	}
-	encoded, err := s.erasure.Split(data)
+	encoded, err := e.encoder.Split(data)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
-	if err = s.erasure.Encode(encoded); err != nil {
+	if err = e.encoder.Encode(encoded); err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
 	return encoded, nil
 }
 
-// ErasureDecodeDataBlocks decodes the given erasure-coded data.
+// DecodeDataBlocks decodes the given erasure-coded data.
 // It only decodes the data blocks but does not verify them.
 // It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
+func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
 	needsReconstruction := false
-	for _, b := range data[:s.dataBlocks] {
+	for _, b := range data[:e.dataBlocks] {
 		if b == nil {
 			needsReconstruction = true
 			break
@@ -82,16 +79,16 @@ func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
 	if !needsReconstruction {
 		return nil
 	}
-	if err := s.erasure.ReconstructData(data); err != nil {
+	if err := e.encoder.ReconstructData(data); err != nil {
 		return err
 	}
 	return nil
 }
 
-// ErasureDecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
+// DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
 // It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
-	if err := s.erasure.Reconstruct(data); err != nil {
+func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
+	if err := e.encoder.Reconstruct(data); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}

cmd/erasure_test.go

@@ -25,7 +25,7 @@ import (
 	"testing"
 )
 
-var erasureDecodeTests = []struct {
+var erasureEncodeDecodeTests = []struct {
 	dataBlocks, parityBlocks   int
 	missingData, missingParity int
 	reconstructParity          bool
@@ -43,20 +43,20 @@ var erasureDecodeTests = []struct {
 	{dataBlocks: 8, parityBlocks: 4, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: false},
 }
 
-func TestErasureDecode(t *testing.T) {
+func TestErasureEncodeDecode(t *testing.T) {
 	data := make([]byte, 256)
 	if _, err := io.ReadFull(rand.Reader, data); err != nil {
 		t.Fatalf("Failed to read random data: %v", err)
 	}
-	for i, test := range erasureDecodeTests {
+	for i, test := range erasureEncodeDecodeTests {
 		buffer := make([]byte, len(data), 2*len(data))
 		copy(buffer, data)
 
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.parityBlocks, blockSizeV1)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.parityBlocks, blockSizeV1)
 		if err != nil {
-			t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
+			t.Fatalf("Test %d: failed to create erasure: %v", i, err)
 		}
-		encoded, err := storage.ErasureEncode(context.Background(), buffer)
+		encoded, err := erasure.EncodeData(context.Background(), buffer)
 		if err != nil {
 			t.Fatalf("Test %d: failed to encode data: %v", i, err)
 		}
@@ -69,9 +69,9 @@ func TestErasureDecode(t *testing.T) {
 		}
 
 		if test.reconstructParity {
-			err = storage.ErasureDecodeDataAndParityBlocks(context.Background(), encoded)
+			err = erasure.DecodeDataAndParityBlocks(context.Background(), encoded)
 		} else {
-			err = storage.ErasureDecodeDataBlocks(encoded)
+			err = erasure.DecodeDataBlocks(encoded)
 		}
 
 		if err == nil && test.shouldFail {

cmd/format-xl.go

@@ -379,6 +379,22 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
 	return disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile)
 }
 
+var ignoredHiddenDirectories = []string{
+	minioMetaBucket,
+	"lost+found",
+	"$RECYCLE.BIN",
+	"System Volume Information",
+}
+
+func isIgnoreHiddenDirectories(dir string) bool {
+	for _, ignDir := range ignoredHiddenDirectories {
+		if dir == ignDir {
+			return true
+		}
+	}
+	return false
+}
+
 // loadFormatXL - loads format.json from disk.
 func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
 	buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
@@ -391,9 +407,7 @@ func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
 			if err != nil {
 				return nil, err
 			}
-			if len(vols) > 1 || (len(vols) == 1 &&
-				vols[0].Name != minioMetaBucket &&
-				vols[0].Name != "lost+found") {
+			if len(vols) > 1 || (len(vols) == 1 && !isIgnoreHiddenDirectories(vols[0].Name)) {
 				// 'format.json' not found, but we
 				// found user data.
 				return nil, errCorruptedFormat

cmd/fs-v1-metadata.go

@@ -24,7 +24,6 @@ import (
 	"io/ioutil"
 	"os"
 	pathutil "path"
-	"strings"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
@@ -137,11 +136,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 
 	// Guess content-type from the extension if possible.
 	if m.Meta["content-type"] == "" {
-		if objectExt := pathutil.Ext(object); objectExt != "" {
-			if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
-				m.Meta["content-type"] = content.ContentType
-			}
-		}
+		m.Meta["content-type"] = mimedb.TypeByExtension(pathutil.Ext(object))
 	}
 
 	if hasSuffix(object, slashSeparator) {
@@ -223,11 +218,13 @@ func parseFSPartsArray(fsMetaBuf []byte) []objectPartInfo {
 		name := gjson.Get(partJSON, "name").String()
 		etag := gjson.Get(partJSON, "etag").String()
 		size := gjson.Get(partJSON, "size").Int()
+		actualSize := gjson.Get(partJSON, "actualSize").Int()
 		partsArray = append(partsArray, objectPartInfo{
-			Number: int(number),
-			Name:   name,
-			ETag:   etag,
-			Size:   size,
+			Number:     int(number),
+			Name:       name,
+			ETag:       etag,
+			Size:       size,
+			ActualSize: int64(actualSize),
 		})
 		return true
 	})

cmd/fs-v1-metadata_test.go

@@ -54,7 +54,7 @@ func TestReadFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -89,7 +89,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 

cmd/fs-v1-multipart.go

@@ -46,21 +46,25 @@ func (fs *FSObjects) getMultipartSHADir(bucket, object string) string {
 }
 
 // Returns partNumber.etag
-func (fs *FSObjects) encodePartFile(partNumber int, etag string) string {
-	return fmt.Sprintf("%.5d.%s", partNumber, etag)
+func (fs *FSObjects) encodePartFile(partNumber int, etag string, actualSize int64) string {
+	return fmt.Sprintf("%.5d.%s.%d", partNumber, etag, actualSize)
 }
 
 // Returns partNumber and etag
-func (fs *FSObjects) decodePartFile(name string) (partNumber int, etag string, err error) {
+func (fs *FSObjects) decodePartFile(name string) (partNumber int, etag string, actualSize int64, err error) {
 	result := strings.Split(name, ".")
-	if len(result) != 2 {
-		return 0, "", errUnexpected
+	if len(result) != 3 {
+		return 0, "", 0, errUnexpected
 	}
 	partNumber, err = strconv.Atoi(result[0])
 	if err != nil {
-		return 0, "", errUnexpected
+		return 0, "", 0, errUnexpected
 	}
-	return partNumber, result[1], nil
+	actualSize, err = strconv.ParseInt(result[2], 10, 64)
+	if err != nil {
+		return 0, "", 0, errUnexpected
+	}
+	return partNumber, result[1], actualSize, nil
 }
 
 // Appends parts to an appendFile sequentially.
@@ -95,7 +99,7 @@ func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploa
 		if entry == fs.metaJSONFile {
 			continue
 		}
-		partNumber, etag, err := fs.decodePartFile(entry)
+		partNumber, etag, actualSize, err := fs.decodePartFile(entry)
 		if err != nil {
 			logger.GetReqInfo(ctx).AppendTags("entry", entry)
 			logger.LogIf(ctx, err)
@@ -119,7 +123,7 @@ func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploa
 			return
 		}
 
-		file.parts = append(file.parts, PartInfo{PartNumber: nextPartNumber, ETag: etag})
+		file.parts = append(file.parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
 		nextPartNumber++
 	}
 }
@@ -209,7 +213,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k
 // subsequent request each UUID is unique.
 //
 // Implements S3 compatible initiate multipart API.
-func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (string, error) {
+func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) (string, error) {
 	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
@@ -249,29 +253,13 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 // object. Internally incoming data is written to '.minio.sys/tmp' location
 // and safely renamed to '.minio.sys/multipart' for reach parts.
 func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int,
-	startOffset int64, length int64, srcInfo ObjectInfo) (pi PartInfo, e error) {
+	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
 
 	if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, fs); err != nil {
 		return pi, toObjectErr(err)
 	}
 
-	// Initialize pipe.
-	go func() {
-		if gerr := fs.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil {
-			if gerr = srcInfo.Writer.Close(); gerr != nil {
-				logger.LogIf(ctx, gerr)
-				return
-			}
-			return
-		}
-		// Close writer explicitly signaling we wrote all data.
-		if gerr := srcInfo.Writer.Close(); gerr != nil {
-			logger.LogIf(ctx, gerr)
-			return
-		}
-	}()
-
-	partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader)
+	partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader, dstOpts)
 	if err != nil {
 		return pi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -283,7 +271,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio.sys/tmp' location and safely renamed to
 // '.minio.sys/multipart' for reach parts.
-func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) {
+func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, e error) {
 	if err := checkPutObjectPartArgs(ctx, bucket, object, fs); err != nil {
 		return pi, toObjectErr(err, bucket)
 	}
@@ -292,8 +280,8 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		return pi, toObjectErr(err, bucket)
 	}
 
-	// Validate input data size and it can never be less than zero.
-	if data.Size() < 0 {
+	// Validate input data size and it can never be less than -1.
+	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return pi, toObjectErr(errInvalidArgument)
 	}
@@ -338,7 +326,7 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	if etag == "" {
 		etag = GenETag()
 	}
-	partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag))
+	partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag, data.ActualSize()))
 
 	if err = fsRenameFile(ctx, tmpPartPath, partPath); err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
@@ -355,6 +343,7 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		LastModified: fi.ModTime(),
 		ETag:         etag,
 		Size:         fi.Size(),
+		ActualSize:   data.ActualSize(),
 	}, nil
 }
 
@@ -381,8 +370,7 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 	}
 
 	uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
-	_, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile))
-	if err != nil {
+	if _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)); err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
 			return result, InvalidUploadID{UploadID: uploadID}
 		}
@@ -400,7 +388,7 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 		if entry == fs.metaJSONFile {
 			continue
 		}
-		partNumber, etag1, derr := fs.decodePartFile(entry)
+		partNumber, etag1, _, derr := fs.decodePartFile(entry)
 		if derr != nil {
 			logger.LogIf(ctx, derr)
 			return result, toObjectErr(derr)
@@ -410,11 +398,11 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 			partsMap[partNumber] = etag1
 			continue
 		}
-		stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag1)))
+		stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag1)))
 		if serr != nil {
 			return result, toObjectErr(serr)
 		}
-		stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag2)))
+		stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag2)))
 		if serr != nil {
 			return result, toObjectErr(serr)
 		}
@@ -424,8 +412,19 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 	}
 
 	var parts []PartInfo
+	var actualSize int64
 	for partNumber, etag := range partsMap {
-		parts = append(parts, PartInfo{PartNumber: partNumber, ETag: etag})
+		partFile := getPartFile(entries, partNumber, etag)
+		if partFile == "" {
+			return result, InvalidPart{}
+		}
+		// Read the actualSize from the pathFileName.
+		subParts := strings.Split(partFile, ".")
+		actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
+		if err != nil {
+			return result, InvalidPart{}
+		}
+		parts = append(parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
 	}
 	sort.Slice(parts, func(i int, j int) bool {
 		return parts[i].PartNumber < parts[j].PartNumber
@@ -455,12 +454,12 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 	}
 	for i, part := range result.Parts {
 		var stat os.FileInfo
-		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag)))
+		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag, part.ActualSize)))
 		if err != nil {
 			return result, toObjectErr(err)
 		}
 		result.Parts[i].LastModified = stat.ModTime()
-		result.Parts[i].Size = stat.Size()
+		result.Parts[i].Size = part.ActualSize
 	}
 
 	fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))
@@ -480,6 +479,9 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 //
 // Implements S3 compatible Complete multipart API.
 func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) {
+
+	var actualSize int64
+
 	if err := checkCompleteMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return oi, toObjectErr(err)
 	}
@@ -516,9 +518,37 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	// Allocate parts similar to incoming slice.
 	fsMeta.Parts = make([]objectPartInfo, len(parts))
 
+	entries, err := readDir(uploadIDDir)
+	if err != nil {
+		logger.GetReqInfo(ctx).AppendTags("uploadIDDir", uploadIDDir)
+		logger.LogIf(ctx, err)
+		return oi, err
+	}
+
+	// Save consolidated actual size.
+	var objectActualSize int64
 	// Validate all parts and then commit to disk.
 	for i, part := range parts {
-		partPath := pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag))
+		partFile := getPartFile(entries, part.PartNumber, part.ETag)
+		if partFile == "" {
+			return oi, InvalidPart{
+				PartNumber: part.PartNumber,
+				GotETag:    part.ETag,
+			}
+		}
+
+		// Read the actualSize from the pathFileName.
+		subParts := strings.Split(partFile, ".")
+		actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
+		if err != nil {
+			return oi, InvalidPart{
+				PartNumber: part.PartNumber,
+				GotETag:    part.ETag,
+			}
+		}
+
+		partPath := pathJoin(uploadIDDir, partFile)
+
 		var fi os.FileInfo
 		fi, err = fsStatFile(ctx, partPath)
 		if err != nil {
@@ -528,24 +558,28 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 			return oi, err
 		}
 		if partSize == -1 {
-			partSize = fi.Size()
+			partSize = actualSize
 		}
 
 		fsMeta.Parts[i] = objectPartInfo{
-			Number: part.PartNumber,
-			ETag:   part.ETag,
-			Size:   fi.Size(),
+			Number:     part.PartNumber,
+			ETag:       part.ETag,
+			Size:       fi.Size(),
+			ActualSize: actualSize,
 		}
 
+		// Consolidate the actual size.
+		objectActualSize += actualSize
+
 		if i == len(parts)-1 {
 			break
 		}
 
 		// All parts except the last part has to be atleast 5MB.
-		if !isMinAllowedPartSize(fi.Size()) {
+		if !isMinAllowedPartSize(actualSize) {
 			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
-				PartSize:   fi.Size(),
+				PartSize:   actualSize,
 				PartETag:   part.ETag,
 			}
 		}
@@ -587,11 +621,12 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	if appendFallback {
-		fsRemoveFile(ctx, file.filePath)
+		if file != nil {
+			fsRemoveFile(ctx, file.filePath)
+		}
 		for _, part := range parts {
-			partPath := pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag))
-			err = mioutil.AppendFile(appendFilePath, partPath)
-			if err != nil {
+			partPath := getPartFile(entries, part.PartNumber, part.ETag)
+			if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partPath)); err != nil {
 				logger.LogIf(ctx, err)
 				return oi, toObjectErr(err)
 			}
@@ -628,6 +663,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		fsMeta.Meta = make(map[string]string)
 	}
 	fsMeta.Meta["etag"] = s3MD5
+	// Save consolidated actual size.
+	fsMeta.Meta[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 	if _, err = fsMeta.WriteTo(metaFile); err != nil {
 		logger.LogIf(ctx, err)
 		return oi, toObjectErr(err, bucket, object)
@@ -676,6 +713,11 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 	}
 
 	fs.appendFileMapMu.Lock()
+	// Remove file in tmp folder
+	file := fs.appendFileMap[uploadID]
+	if file != nil {
+		fsRemoveFile(ctx, file.filePath)
+	}
 	delete(fs.appendFileMap, uploadID)
 	fs.appendFileMapMu.Unlock()
 
@@ -688,10 +730,14 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 		}
 		return toObjectErr(err, bucket, object)
 	}
+
 	// Ignore the error returned as Windows fails to remove directory if a file in it
 	// is Open()ed by the backgroundAppend()
 	fsRemoveAll(ctx, uploadIDDir)
 
+	// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
+	fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
+
 	return nil
 }
 

cmd/fs-v1-multipart_test.go

@@ -42,7 +42,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
@@ -81,7 +81,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 
 	// Test with disk removed.
 	os.RemoveAll(disk)
-	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}); err != nil {
+	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -105,7 +105,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -114,7 +114,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	sha256sum := ""
 
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum))
+	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
 	if !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -136,7 +136,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -168,14 +168,14 @@ func TestCompleteMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
+	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
@@ -202,14 +202,14 @@ func TestAbortMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
+	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
@@ -234,7 +234,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}

cmd/fs-v1.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"sort"
@@ -261,7 +263,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
 	storageInfo := StorageInfo{
 		Used: used,
 	}
-	storageInfo.Backend.Type = FS
+	storageInfo.Backend.Type = BackendFS
 	return storageInfo
 }
 
@@ -414,37 +416,21 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
 // CopyObject - copy object source object to destination object.
 // if source object and destination object are same we only
 // update metadata.
-func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo) (oi ObjectInfo, e error) {
+func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
-	// Hold write lock on destination since in both cases
-	// - if source and destination are same
-	// - if source and destination are different
-	// it is the sole mutating state.
-	objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
-	if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
-		return oi, err
-	}
-	defer objectDWLock.Unlock()
-	// if source and destination are different, we have to hold
-	// additional read lock as well to protect against writes on
-	// source.
 	if !cpSrcDstSame {
-		// Hold read locks on source object only if we are
-		// going to read data from source object.
-		objectSRLock := fs.nsMutex.NewNSLock(srcBucket, srcObject)
-		if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
+		objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
+		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, err
 		}
-		defer objectSRLock.RUnlock()
+		defer objectDWLock.Unlock()
 	}
+
 	if _, err := fs.statBucketDir(ctx, srcBucket); err != nil {
 		return oi, toObjectErr(err, srcBucket)
 	}
 
 	if cpSrcDstSame && srcInfo.metadataOnly {
-		// Close any writer which was initialized.
-		defer srcInfo.Writer.Close()
-
 		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, srcObject, fs.metaJSONFile)
 		wlk, err := fs.rwPool.Write(fsMetaPath)
 		if err != nil {
@@ -456,7 +442,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 
 		// Save objects' metadata in `fs.json`.
 		fsMeta := newFSMetaV1()
-		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil {
+		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil && err != io.EOF {
 			return oi, toObjectErr(err, srcBucket, srcObject)
 		}
 
@@ -476,20 +462,6 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
 
-	go func() {
-		if gerr := fs.getObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, !cpSrcDstSame); gerr != nil {
-			if gerr = srcInfo.Writer.Close(); gerr != nil {
-				logger.LogIf(ctx, gerr)
-			}
-			return
-		}
-		// Close writer explicitly signaling we wrote all data.
-		if gerr := srcInfo.Writer.Close(); gerr != nil {
-			logger.LogIf(ctx, gerr)
-			return
-		}
-	}()
-
 	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.Reader, srcInfo.UserDefined)
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
@@ -498,13 +470,107 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 	return objInfo, nil
 }
 
+// GetObjectNInfo - returns object info and a reader for object
+// content.
+func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+
+	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
+		return nil, err
+	}
+
+	if _, err = fs.statBucketDir(ctx, bucket); err != nil {
+		return nil, toObjectErr(err, bucket)
+	}
+
+	var nsUnlocker = func() {}
+
+	if lockType != noLock {
+		// Lock the object before reading.
+		lock := fs.nsMutex.NewNSLock(bucket, object)
+		switch lockType {
+		case writeLock:
+			if err = lock.GetLock(globalObjectTimeout); err != nil {
+				logger.LogIf(ctx, err)
+				return nil, err
+			}
+			nsUnlocker = lock.Unlock
+		case readLock:
+			if err = lock.GetRLock(globalObjectTimeout); err != nil {
+				logger.LogIf(ctx, err)
+				return nil, err
+			}
+			nsUnlocker = lock.RUnlock
+		}
+	}
+
+	// For a directory, we need to send an reader that returns no bytes.
+	if hasSuffix(object, slashSeparator) {
+		// The lock taken above is released when
+		// objReader.Close() is called by the caller.
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), ObjectInfo{}, nsUnlocker), nil
+	}
+
+	// Otherwise we get the object info
+	var objInfo ObjectInfo
+	if objInfo, err = fs.getObjectInfo(ctx, bucket, object); err != nil {
+		nsUnlocker()
+		return nil, toObjectErr(err, bucket, object)
+	}
+
+	// Take a rwPool lock for NFS gateway type deployment
+	rwPoolUnlocker := func() {}
+	if bucket != minioMetaBucket && lockType != noLock {
+		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+		_, err = fs.rwPool.Open(fsMetaPath)
+		if err != nil && err != errFileNotFound {
+			logger.LogIf(ctx, err)
+			nsUnlocker()
+			return nil, toObjectErr(err, bucket, object)
+		}
+		// Need to clean up lock after getObject is
+		// completed.
+		rwPoolUnlocker = func() { fs.rwPool.Close(fsMetaPath) }
+	}
+
+	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, nsUnlocker, rwPoolUnlocker)
+	if rErr != nil {
+		return nil, rErr
+	}
+
+	// Read the object, doesn't exist returns an s3 compatible error.
+	fsObjPath := pathJoin(fs.fsPath, bucket, object)
+	readCloser, size, err := fsOpenFile(ctx, fsObjPath, off)
+	if err != nil {
+		rwPoolUnlocker()
+		nsUnlocker()
+		return nil, toObjectErr(err, bucket, object)
+	}
+	var reader io.Reader
+	reader = io.LimitReader(readCloser, length)
+	closeFn := func() {
+		readCloser.Close()
+	}
+
+	// Check if range is valid
+	if off > size || off+length > size {
+		err = InvalidRange{off, length, size}
+		logger.LogIf(ctx, err)
+		closeFn()
+		rwPoolUnlocker()
+		nsUnlocker()
+		return nil, err
+	}
+
+	return objReaderFn(reader, h, closeFn)
+}
+
 // GetObject - reads an object from the disk.
 // Supports additional parameters like offset and length
 // which are synonymous with HTTP Range requests.
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) {
+func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
 		return err
 	}
@@ -596,7 +662,10 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
 	buf := make([]byte, int(bufSize))
 
 	_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
-	logger.LogIf(ctx, err)
+	// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
+	if err == io.ErrClosedPipe {
+		err = nil
+	}
 	return toObjectErr(err, bucket, object)
 }
 
@@ -605,13 +674,8 @@ func (fs *FSObjects) createFsJSON(object, fsMetaPath string) error {
 	fsMeta := newFSMetaV1()
 	fsMeta.Meta = make(map[string]string)
 	fsMeta.Meta["etag"] = GenETag()
-	if objectExt := path.Ext(object); objectExt != "" {
-		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
-			fsMeta.Meta["content-type"] = content.ContentType
-		} else {
-			fsMeta.Meta["content-type"] = "application/octet-stream"
-		}
-	}
+	contentType := mimedb.TypeByExtension(path.Ext(object))
+	fsMeta.Meta["content-type"] = contentType
 	wlk, werr := fs.rwPool.Create(fsMetaPath)
 	if werr == nil {
 		_, err := fsMeta.WriteTo(wlk)
@@ -626,13 +690,8 @@ func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 {
 	fsMeta := newFSMetaV1()
 	fsMeta.Meta = make(map[string]string)
 	fsMeta.Meta["etag"] = defaultEtag
-	if objectExt := path.Ext(object); objectExt != "" {
-		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
-			fsMeta.Meta["content-type"] = content.ContentType
-		} else {
-			fsMeta.Meta["content-type"] = "application/octet-stream"
-		}
-	}
+	contentType := mimedb.TypeByExtension(path.Ext(object))
+	fsMeta.Meta["content-type"] = contentType
 	return fsMeta
 }
 
@@ -660,7 +719,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 		// Read from fs metadata only if it exists.
 		_, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile)
 		fs.rwPool.Close(fsMetaPath)
-		if rerr != nil {
+		if rerr != nil && rerr != io.EOF {
 			return oi, rerr
 		}
 	}
@@ -710,7 +769,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 }
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
-func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
 		objectLock := fs.nsMutex.NewNSLock(bucket, object)
@@ -754,7 +813,7 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	if err := checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
 		return ObjectInfo{}, err
 	}
@@ -816,7 +875,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	}
 
 	// Validate input data size and it can never be less than zero.
-	if data.Size() < 0 {
+	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return ObjectInfo{}, errInvalidArgument
 	}
@@ -959,7 +1018,7 @@ func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
 	listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
-		if err != nil {
+		if err != nil && err != errFileNotFound {
 			logger.LogIf(context.Background(), err)
 			return
 		}
@@ -1215,7 +1274,7 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 
 // SetBucketPolicy sets policy on bucket
 func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
-	return savePolicyConfig(fs, bucket, policy)
+	return savePolicyConfig(ctx, fs, bucket, policy)
 }
 
 // GetBucketPolicy will get policy on bucket
@@ -1259,3 +1318,8 @@ func (fs *FSObjects) IsNotificationSupported() bool {
 func (fs *FSObjects) IsEncryptionSupported() bool {
 	return true
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (fs *FSObjects) IsCompressionSupported() bool {
+	return true
+}

cmd/fs-v1_test.go

@@ -40,7 +40,7 @@ func TestFSParentDirIsObject(t *testing.T) {
 	}
 	objectContent := "12345"
 	objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
-		mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
+		mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,7 +124,7 @@ func TestFSShutdown(t *testing.T) {
 
 		objectContent := "12345"
 		obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-		obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
+		obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
 		return fs, disk
 	}
 
@@ -203,7 +203,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a regular object.
-	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -212,7 +212,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -220,11 +220,11 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
@@ -239,7 +239,7 @@ func TestFSPutObject(t *testing.T) {
 		}
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
@@ -267,7 +267,7 @@ func TestFSDeleteObject(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 
 	// Test with invalid bucket name
 	if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) {

cmd/gateway-common.go

@@ -224,7 +224,16 @@ func FromMinioClientListBucketResultToV2Info(bucket string, result minio.ListBuc
 	}
 }
 
-// ToMinioClientMetadata converts metadata to map[string][]string
+// ToMinioClientObjectInfoMetadata convertes metadata to map[string][]string
+func ToMinioClientObjectInfoMetadata(metadata map[string]string) map[string][]string {
+	mm := make(map[string][]string, len(metadata))
+	for k, v := range metadata {
+		mm[http.CanonicalHeaderKey(k)] = []string{v}
+	}
+	return mm
+}
+
+// ToMinioClientMetadata converts metadata to map[string]string
 func ToMinioClientMetadata(metadata map[string]string) map[string]string {
 	mm := make(map[string]string)
 	for k, v := range metadata {
@@ -285,6 +294,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = BucketPolicyNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
+	case "InvalidPart":
+		err = InvalidPart{}
 	case "NoSuchBucket":
 		err = BucketNotFound{Bucket: bucket}
 	case "NoSuchKey":

cmd/gateway-main.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"os"
 	"os/signal"
@@ -133,63 +134,53 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
 
-	// Handle common env vars.
-	handleCommonEnvVars()
+	// Get port to listen on from gateway address
+	_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
+	if pErr != nil {
+		logger.FatalIf(pErr, "Unable to start gateway")
+	}
 
 	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
 	// to IPv6 address ie minio will start listening on IPv6 address whereas another
 	// (non-)minio process is listening on IPv4 of given port.
 	// To avoid this error situation we check for port availability.
-	logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the server")
+	logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
+
+	// Create certs path.
+	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
+
+	// Check and load TLS certificates.
+	var err error
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
+	logger.FatalIf(err, "Invalid TLS certificate file")
+
+	// Check and load Root CAs.
+	globalRootCAs, err = getRootCAs(getCADir())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Handle common env vars.
+	handleCommonEnvVars()
 
 	// Validate if we have access, secret set through environment.
 	if !globalIsEnvCreds {
 		logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
 	}
 
-	// Create certs path.
-	logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
-
-	// Initialize server config.
-	srvCfg := newServerConfig()
-
-	// Override any values from ENVs.
-	srvCfg.loadFromEnvs()
-
-	// Load values to cached global values.
-	srvCfg.loadToCachedConfigs()
-
-	// hold the mutex lock before a new config is assigned.
-	globalServerConfigMu.Lock()
-	globalServerConfig = srvCfg
-	globalServerConfigMu.Unlock()
-
-	var cacheConfig = globalServerConfig.GetCacheConfig()
-	if len(cacheConfig.Drives) > 0 {
-		var err error
-		// initialize the new disk cache objects.
-		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
-		logger.FatalIf(err, "Unable to initialize disk caching")
-	}
-
-	// Check and load SSL certificates.
-	var err error
-	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
-	logger.FatalIf(err, "Invalid SSL certificate file")
-
 	// Set system resources to maximum.
 	logger.LogIf(context.Background(), setMaxResources())
 
 	initNSLock(false) // Enable local namespace lock.
 
-	// Create new notification system.
-	globalNotificationSys = NewNotificationSys(globalServerConfig, EndpointList{})
-
-	// Create new policy system.
-	globalPolicySys = NewPolicySys()
-
 	router := mux.NewRouter().SkipClean(true)
 
+	if globalEtcdClient != nil {
+		// Enable STS router if etcd is enabled.
+		registerSTSRouter(router)
+
+		// Enable admin router if etcd is enabled.
+		registerAdminRouter(router)
+	}
+
 	// Add healthcheck router
 	registerHealthCheckRouter(router)
 
@@ -227,8 +218,56 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 
+	// Create a new config system.
+	globalConfigSys = NewConfigSys()
+
+	// Initialize server config.
+	srvCfg := newServerConfig()
+
+	// Override any values from ENVs.
+	srvCfg.loadFromEnvs()
+
+	// Load values to cached global values.
+	srvCfg.loadToCachedConfigs()
+
+	// hold the mutex lock before a new config is assigned.
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()
+
+	// Load logger subsystem
+	loadLoggers()
+
+	// This is only to uniquely identify each gateway deployments.
+	logger.SetDeploymentID(os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID"))
+
+	var cacheConfig = globalServerConfig.GetCacheConfig()
+	if len(cacheConfig.Drives) > 0 {
+		var err error
+		// initialize the new disk cache objects.
+		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
+		logger.FatalIf(err, "Unable to initialize disk caching")
+	}
+
+	// Re-enable logging
+	logger.Disable = false
+
+	// Create new IAM system.
+	globalIAMSys = NewIAMSys()
+	if globalEtcdClient != nil {
+		// Initialize IAM sys.
+		go globalIAMSys.Init(newObject)
+	}
+
+	// Create new policy system.
+	globalPolicySys = NewPolicySys()
+
+	// Initialize policy system.
 	go globalPolicySys.Init(newObject)
 
+	// Create new notification system.
+	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
+
 	// Once endpoints are finalized, initialize the new object api.
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = newObject
@@ -249,8 +288,5 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
 	}
 
-	// Reenable logging
-	logger.Disable = false
-
 	handleSignals()
 }

cmd/gateway-startup-msg.go

@@ -57,8 +57,10 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 	// Colorize the message and print.
 	logger.StartupMessage(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
-	logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
-	logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+	if isTerminal() {
+		logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
+	}
 
 	if globalIsBrowserEnabled {
 		logger.StartupMessage(colorBlue("\nBrowser Access:"))

cmd/gateway-unsupported.go

@@ -35,19 +35,19 @@ func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket str
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) {
+func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return "", NotImplemented{}
 }
 
 // CopyObjectPart copy part of object to uploadID for another object
-func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo) (pi PartInfo, err error) {
+func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return pi, NotImplemented{}
 }
 
 // PutObjectPart puts a part of object in bucket
-func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) {
+func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return pi, NotImplemented{}
 }
@@ -131,7 +131,7 @@ func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix,
 
 // CopyObject copies a blob from source container to destination container.
 func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
-	srcInfo ObjectInfo) (objInfo ObjectInfo, err error) {
+	srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return objInfo, NotImplemented{}
 }
@@ -151,3 +151,8 @@ func (a GatewayUnsupported) IsNotificationSupported() bool {
 func (a GatewayUnsupported) IsEncryptionSupported() bool {
 	return false
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (a GatewayUnsupported) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/azure/gateway-azure.go

@@ -615,13 +615,38 @@ func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 	return result, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (a *azureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = a.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := a.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject - reads an object from azure. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	// startOffset cannot be negative.
 	if startOffset < 0 {
 		return azureToObjectError(minio.InvalidRange{}, bucket, object)
@@ -653,17 +678,40 @@ func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, sta
 
 // GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo,
 // uses zure equivalent GetBlobProperties.
-func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	err = blob.GetProperties(nil)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
 
+	// Populate correct ETag's if possible, this code primarily exists
+	// because AWS S3 indicates that
+	//
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+	//
+	// Objects created by the PUT Object, POST Object, or Copy operation,
+	// or through the AWS Management Console, and are encrypted by SSE-S3
+	// or plaintext, have ETags that are an MD5 digest of their object data.
+	//
+	// Some applications depend on this behavior refer https://github.com/minio/minio/issues/6550
+	// So we handle it here and make this consistent.
+	etag := minio.ToS3ETag(blob.Properties.Etag)
+	switch {
+	case blob.Properties.ContentMD5 != "":
+		b, err := base64.StdEncoding.DecodeString(blob.Properties.ContentMD5)
+		if err == nil {
+			etag = hex.EncodeToString(b)
+		}
+	case blob.Metadata["md5sum"] != "":
+		etag = blob.Metadata["md5sum"]
+		delete(blob.Metadata, "md5sum")
+	}
+
 	return minio.ObjectInfo{
 		Bucket:          bucket,
 		UserDefined:     azurePropertiesToS3Meta(blob.Metadata, blob.Properties),
-		ETag:            minio.ToS3ETag(blob.Properties.Etag),
+		ETag:            etag,
 		ModTime:         time.Time(blob.Properties.LastModified),
 		Name:            object,
 		Size:            blob.Properties.ContentLength,
@@ -674,22 +722,110 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
-func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if data.Size() < azureBlockSize/10 {
+		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
+		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+		if err = blob.CreateBlockBlobFromReader(data, nil); err != nil {
+			return objInfo, azureToObjectError(err, bucket, object)
+		}
+		return a.GetObjectInfo(ctx, bucket, object, opts)
+	}
+
+	uuid, err := getAzureUploadID()
+	if err != nil {
+		return objInfo, err
+	}
+	etag := data.MD5HexString()
+	if etag == "" {
+		etag = minio.GenETag()
+	}
+
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-	blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+	subPartSize, subPartNumber := int64(azureBlockSize), 1
+	for remainingSize := data.Size(); remainingSize >= 0; remainingSize -= subPartSize {
+		// Allow to create zero sized part.
+		if remainingSize == 0 && subPartNumber > 1 {
+			break
+		}
+
+		if remainingSize < subPartSize {
+			subPartSize = remainingSize
+		}
+
+		id := azureGetBlockID(1, subPartNumber, uuid, etag)
+		if err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil); err != nil {
+			return objInfo, azureToObjectError(err, bucket, object)
+		}
+		subPartNumber++
+	}
+
+	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
+	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	err = blob.CreateBlockBlobFromReader(data, nil)
+
+	getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, aerr error) {
+		for _, part := range resp.UncommittedBlocks {
+			var partID int
+			var readUploadID string
+			var md5Hex string
+			if partID, _, readUploadID, md5Hex, aerr = azureParseBlockID(part.Name); aerr != nil {
+				return nil, 0, aerr
+			}
+
+			if partNumber == partID && uuid == readUploadID && etag == md5Hex {
+				blocks = append(blocks, storage.Block{
+					ID:     part.Name,
+					Status: storage.BlockStatusUncommitted,
+				})
+
+				size += part.Size
+			}
+		}
+
+		if len(blocks) == 0 {
+			return nil, 0, minio.InvalidPart{}
+		}
+
+		return blocks, size, nil
+	}
+
+	var blocks []storage.Block
+	blocks, _, err = getBlocks(1, etag)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, err
+	}
+
+	if err = objBlob.PutBlockList(blocks, nil); err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+
+	if len(metadata) == 0 {
+		metadata = map[string]string{}
+	}
+
+	// Save md5sum for future processing on the object.
+	metadata["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
+	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	return a.GetObjectInfo(ctx, bucket, object)
+	if err = objBlob.SetProperties(nil); err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+	if err = objBlob.SetMetadata(nil); err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+
+	return a.GetObjectInfo(ctx, bucket, object, opts)
 }
 
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Azure equivalent CopyBlob API.
-func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
 	destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
 	azureMeta, props, err := s3MetaToAzureProperties(ctx, srcInfo.UserDefined)
@@ -716,7 +852,7 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des
 	if err != nil {
 		return objInfo, azureToObjectError(err, srcBucket, srcObject)
 	}
-	return a.GetObjectInfo(ctx, destBucket, destObject)
+	return a.GetObjectInfo(ctx, destBucket, destObject, dstOpts)
 }
 
 // DeleteObject - Deletes a blob on azure container, uses Azure
@@ -763,7 +899,7 @@ func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, obje
 }
 
 // NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
-func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts minio.ObjectOptions) (uploadID string, err error) {
 	uploadID, err = getAzureUploadID()
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -787,7 +923,7 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
 }
 
 // PutObjectPart - Use Azure equivalent PutBlockWithLength.
-func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info minio.PartInfo, err error) {
+func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return info, err
 	}
@@ -949,10 +1085,6 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	}
 
 	defer func() {
-		if err != nil {
-			return
-		}
-
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
 		derr := blob.Delete(nil)
 		logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
@@ -1035,7 +1167,7 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
 	}
-	return a.GetObjectInfo(ctx, bucket, object)
+	return a.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy - Azure supports three types of container policies:
@@ -1123,3 +1255,8 @@ func (a *azureObjects) DeleteBucketPolicy(ctx context.Context, bucket string) er
 	err := container.SetPermissions(perm, nil)
 	return azureToObjectError(err)
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (a *azureObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/b2/gateway-b2.go

@@ -23,6 +23,7 @@ import (
 	"hash"
 	"io"
 	"io/ioutil"
+	"net/http"
 
 	"strings"
 	"sync"
@@ -394,13 +395,38 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	return loi, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject reads an object from B2. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return err
@@ -417,7 +443,7 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string,
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
-func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) {
+func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return objInfo, err
@@ -508,7 +534,7 @@ func (nb *Reader) Read(p []byte) (int, error) {
 }
 
 // PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
-func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return objInfo, err
@@ -608,7 +634,7 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 // Each large file must consist of at least 2 parts, and all of the parts except the
 // last one must be at least 5MB in size. The last part must contain at least one byte.
 // For more information - https://www.backblaze.com/b2/docs/large_files.html
-func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (string, error) {
+func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (string, error) {
 	var uploadID string
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
@@ -627,7 +653,7 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 }
 
 // PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
-func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) {
+func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return pi, err
@@ -723,7 +749,7 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, b2ToObjectError(err, bucket, object, uploadID)
 	}
 
-	return l.GetObjectInfo(ctx, bucket, object)
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy - B2 supports 2 types of bucket policies:
@@ -814,3 +840,8 @@ func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err)
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *b2Objects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/gcs/gateway-gcs.go

@@ -20,11 +20,15 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math"
 	"net/http"
+	"path"
+	"strconv"
+
 	"os"
 	"regexp"
 	"strings"
@@ -732,13 +736,38 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
 	}, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject - reads an object from GCS. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
 	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@@ -746,7 +775,13 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s
 		return gcsToObjectError(err, bucket)
 	}
 
-	object := l.client.Bucket(bucket).Object(key)
+	// GCS storage decompresses a gzipped object by default and returns the data.
+	// Refer to https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding
+	// Need to set `Accept-Encoding` header to `gzip` when issuing a GetObject call, to be able
+	// to download the object in compressed state.
+	// Calling ReadCompressed with true accomplishes that.
+	object := l.client.Bucket(bucket).Object(key).ReadCompressed(true)
+
 	r, err := object.NewRangeReader(l.ctx, startOffset, length)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -827,7 +862,7 @@ func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAt
 }
 
 // GetObjectInfo - reads object info and replies back ObjectInfo
-func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) {
+func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
 	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@@ -845,7 +880,7 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st
 }
 
 // PutObject - Create a new object with the incoming data,
-func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string) (minio.ObjectInfo, error) {
+func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
 	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@@ -884,7 +919,7 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
 
 // CopyObject - Copies a blob from source container to destination container.
 func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
-	srcInfo minio.ObjectInfo) (minio.ObjectInfo, error) {
+	srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (minio.ObjectInfo, error) {
 
 	src := l.client.Bucket(srcBucket).Object(srcObject)
 	dst := l.client.Bucket(destBucket).Object(destObject)
@@ -913,7 +948,7 @@ func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object str
 }
 
 // NewMultipartUpload - upload object in multiple parts
-func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string) (uploadID string, err error) {
+func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
 	// generate new uploadid
 	uploadID = minio.MustGetUUID()
 
@@ -936,14 +971,84 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
 	return uploadID, nil
 }
 
-// ListMultipartUploads - lists all multipart uploads.
+// ListMultipartUploads - lists the (first) multipart upload for an object
+// matched _exactly_ by the prefix
 func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (minio.ListMultipartsInfo, error) {
+	// List objects under <bucket>/gcsMinioMultipartPathV1
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
+		Prefix: gcsMinioMultipartPathV1,
+	})
+
+	var uploads []minio.MultipartInfo
+
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		}
+
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return minio.ListMultipartsInfo{
+				KeyMarker:      keyMarker,
+				UploadIDMarker: uploadIDMarker,
+				MaxUploads:     maxUploads,
+				Prefix:         prefix,
+				Delimiter:      delimiter,
+			}, gcsToObjectError(err)
+		}
+
+		// Skip entries other than gcs.json
+		if !strings.HasSuffix(attrs.Name, gcsMinioMultipartMeta) {
+			continue
+		}
+
+		// Extract multipart upload information from gcs.json
+		obj := l.client.Bucket(bucket).Object(attrs.Name)
+		objReader, rErr := obj.NewReader(ctx)
+		if rErr != nil {
+			logger.LogIf(ctx, rErr)
+			return minio.ListMultipartsInfo{}, rErr
+		}
+		defer objReader.Close()
+
+		var mpMeta gcsMultipartMetaV1
+		dec := json.NewDecoder(objReader)
+		decErr := dec.Decode(&mpMeta)
+		if decErr != nil {
+			logger.LogIf(ctx, decErr)
+			return minio.ListMultipartsInfo{}, decErr
+		}
+
+		if prefix == mpMeta.Object {
+			// Extract uploadId
+			// E.g minio.sys.tmp/multipart/v1/d063ad89-fdc4-4ea3-a99e-22dba98151f5/gcs.json
+			components := strings.SplitN(attrs.Name, "/", 5)
+			if len(components) != 5 {
+				compErr := errors.New("Invalid multipart upload format")
+				logger.LogIf(ctx, compErr)
+				return minio.ListMultipartsInfo{}, compErr
+			}
+			upload := minio.MultipartInfo{
+				Object:    mpMeta.Object,
+				UploadID:  components[3],
+				Initiated: attrs.Created,
+			}
+			uploads = []minio.MultipartInfo{upload}
+			break
+		}
+	}
+
 	return minio.ListMultipartsInfo{
-		KeyMarker:      keyMarker,
-		UploadIDMarker: uploadIDMarker,
-		MaxUploads:     maxUploads,
-		Prefix:         prefix,
-		Delimiter:      delimiter,
+		KeyMarker:          keyMarker,
+		UploadIDMarker:     uploadIDMarker,
+		MaxUploads:         maxUploads,
+		Prefix:             prefix,
+		Delimiter:          delimiter,
+		Uploads:            uploads,
+		NextKeyMarker:      "",
+		NextUploadIDMarker: "",
+		IsTruncated:        false,
 	}, nil
 }
 
@@ -956,7 +1061,7 @@ func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key
 }
 
 // PutObjectPart puts a part of object in bucket
-func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (minio.PartInfo, error) {
+func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader, opts minio.ObjectOptions) (minio.PartInfo, error) {
 	if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
 		return minio.PartInfo{}, err
 	}
@@ -987,9 +1092,91 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
 
 }
 
-// ListObjectParts returns all object parts for specified object in specified bucket
+// gcsGetPartInfo returns PartInfo of a given object part
+func gcsGetPartInfo(ctx context.Context, attrs *storage.ObjectAttrs) (minio.PartInfo, error) {
+	components := strings.SplitN(attrs.Name, "/", 5)
+	if len(components) != 5 {
+		logger.LogIf(ctx, errors.New("Invalid multipart upload format"))
+		return minio.PartInfo{}, errors.New("Invalid multipart upload format")
+	}
+
+	partComps := strings.SplitN(components[4], ".", 2)
+	if len(partComps) != 2 {
+		logger.LogIf(ctx, errors.New("Invalid multipart part format"))
+		return minio.PartInfo{}, errors.New("Invalid multipart part format")
+	}
+
+	partNum, pErr := strconv.Atoi(partComps[0])
+	if pErr != nil {
+		logger.LogIf(ctx, pErr)
+		return minio.PartInfo{}, errors.New("Invalid part number")
+	}
+
+	return minio.PartInfo{
+		PartNumber:   partNum,
+		LastModified: attrs.Updated,
+		Size:         attrs.Size,
+		ETag:         partComps[1],
+	}, nil
+}
+
+//  ListObjectParts returns all object parts for specified object in specified bucket
 func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) {
-	return minio.ListPartsInfo{}, l.checkUploadIDExists(ctx, bucket, key, uploadID)
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
+		Prefix: path.Join(gcsMinioMultipartPathV1, uploadID),
+	})
+
+	var (
+		count     int
+		partInfos []minio.PartInfo
+	)
+
+	isTruncated := true
+	for count < maxParts {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			isTruncated = false
+			break
+		}
+
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return minio.ListPartsInfo{}, gcsToObjectError(err)
+		}
+
+		if strings.HasSuffix(attrs.Name, gcsMinioMultipartMeta) {
+			continue
+		}
+
+		partInfo, pErr := gcsGetPartInfo(ctx, attrs)
+		if pErr != nil {
+			logger.LogIf(ctx, pErr)
+			return minio.ListPartsInfo{}, pErr
+		}
+
+		if partInfo.PartNumber <= partNumberMarker {
+			continue
+		}
+
+		partInfos = append(partInfos, partInfo)
+		count++
+	}
+
+	nextPartNumberMarker := 0
+	if isTruncated {
+		nextPartNumberMarker = partInfos[maxParts-1].PartNumber
+	}
+
+	return minio.ListPartsInfo{
+		Bucket:               bucket,
+		Object:               key,
+		UploadID:             uploadID,
+		PartNumberMarker:     partNumberMarker,
+		NextPartNumberMarker: nextPartNumberMarker,
+		MaxParts:             maxParts,
+		Parts:                partInfos,
+		IsTruncated:          isTruncated,
+	}, nil
 }
 
 // Called by AbortMultipartUpload and CompleteMultipartUpload for cleaning up.
@@ -1266,3 +1453,8 @@ func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *gcsGateway) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/manta/gateway-manta.go

@@ -117,12 +117,16 @@ EXAMPLES:
 }
 
 func mantaGatewayMain(ctx *cli.Context) {
-	// Validate gateway arguments.
-	host := ctx.Args().First()
-	// Validate gateway arguments.
-	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
+	args := ctx.Args()
+	if !ctx.Args().Present() {
+		args = cli.Args{"https://us-east.manta.joyent.com"}
+	}
 
-	minio.StartGateway(ctx, &Manta{host})
+	// Validate gateway arguments.
+	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), args.First()), "Invalid argument")
+
+	// Start the gateway..
+	minio.StartGateway(ctx, &Manta{args.First()})
 }
 
 // Manta implements Gateway.
@@ -139,17 +143,22 @@ func (g *Manta) Name() string {
 // talk to manta remote backend.
 func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
 	var err error
+	var secure bool
 	var signer authentication.Signer
 	var endpoint = defaultMantaURL
 	ctx := context.Background()
 
 	if g.host != "" {
-		endpoint, _, err = minio.ParseGatewayEndpoint(g.host)
+		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
 		if err != nil {
 			return nil, err
 		}
+		if secure {
+			endpoint = "https://" + endpoint
+		} else {
+			endpoint = "http://" + endpoint
+		}
 	}
-
 	if overrideRoot, ok := os.LookupEnv("MANTA_ROOT"); ok {
 		mantaRoot = overrideRoot
 	}
@@ -497,6 +506,31 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 	return result, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (t *tritonObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = t.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := t.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject - Reads an object from Manta. Supports additional parameters like
 // offset and length which are synonymous with HTTP Range requests.
 //
@@ -504,7 +538,7 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 // indicates the total length of the object.
 //
 // https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	// Start offset cannot be negative.
 	if startOffset < 0 {
 		logger.LogIf(ctx, fmt.Errorf("Unexpected error"))
@@ -536,7 +570,7 @@ func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, st
 // uses Triton equivalent GetBlobProperties.
 //
 // https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	info, err := t.client.Objects().GetInfo(ctx, &storage.GetInfoInput{
 		ObjectPath: path.Join(mantaRoot, bucket, object),
 	})
@@ -574,7 +608,7 @@ func (d dummySeeker) Seek(offset int64, whence int) (int64, error) {
 // CreateBlockBlobFromReader.
 //
 // https://apidocs.joyent.com/manta/api.html#PutObject
-func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{
 		ContentLength: uint64(data.Size()),
 		ObjectPath:    path.Join(mantaRoot, bucket, object),
@@ -593,14 +627,14 @@ func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, da
 		return objInfo, err
 	}
 
-	return t.GetObjectInfo(ctx, bucket, object)
+	return t.GetObjectInfo(ctx, bucket, object, opts)
 }
 
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Manta Snaplinks API.
 //
 // https://apidocs.joyent.com/manta/api.html#PutSnapLink
-func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{
 		SourcePath: path.Join(mantaRoot, srcBucket, srcObject),
 		LinkPath:   path.Join(mantaRoot, destBucket, destObject),
@@ -609,7 +643,7 @@ func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, de
 		return objInfo, err
 	}
 
-	return t.GetObjectInfo(ctx, destBucket, destObject)
+	return t.GetObjectInfo(ctx, destBucket, destObject, dstOpts)
 }
 
 // DeleteObject - Delete a blob in Manta, uses Triton equivalent DeleteBlob API.
@@ -625,3 +659,8 @@ func (t *tritonObjects) DeleteObject(ctx context.Context, bucket, object string)
 
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (t *tritonObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/nas/gateway-nas.go

@@ -125,3 +125,8 @@ type nasObjects struct {
 func (l *nasObjects) IsNotificationSupported() bool {
 	return false
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *nasObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/oss/gateway-oss.go

@@ -21,7 +21,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
@@ -69,7 +68,7 @@ ENVIRONMENT VARIABLES:
 
   DOMAIN:
      MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
-	
+
   CACHE:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
@@ -547,13 +546,38 @@ func ossGetObject(ctx context.Context, client *oss.Client, bucket, key string, s
 	return nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *ossObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject reads an object on OSS. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
+func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	return ossGetObject(ctx, l.Client, bucket, key, startOffset, length, writer, etag)
 }
 
@@ -603,7 +627,7 @@ func ossGetObjectInfo(ctx context.Context, client *oss.Client, bucket, object st
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo.
-func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	return ossGetObjectInfo(ctx, l.Client, bucket, object)
 }
 
@@ -631,12 +655,12 @@ func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string
 }
 
 // PutObject creates a new object with the incoming data.
-func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	return ossPutObject(ctx, l.Client, bucket, object, data, metadata)
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
-func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Client.Bucket(srcBucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -660,7 +684,7 @@ func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		logger.LogIf(ctx, err)
 		return objInfo, ossToObjectError(err, srcBucket, srcObject)
 	}
-	return l.GetObjectInfo(ctx, dstBucket, dstObject)
+	return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts)
 }
 
 // DeleteObject deletes a blob in bucket.
@@ -726,7 +750,7 @@ func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, k
 }
 
 // NewMultipartUpload upload object in multiple parts.
-func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
 	bkt, err := l.Client.Bucket(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -749,7 +773,7 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 }
 
 // PutObjectPart puts a part of object in bucket.
-func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, err error) {
+func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
 	bkt, err := l.Client.Bucket(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -816,11 +840,8 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 		return lupr, err
 	}
 
-	defer func() {
-		// always drain output (response body)
-		io.CopyN(ioutil.Discard, resp.Body, 512)
-		resp.Body.Close()
-	}()
+	// always drain output (response body)
+	defer minio.CloseResponse(resp.Body)
 
 	err = xml.NewDecoder(resp.Body).Decode(&lupr)
 	if err != nil {
@@ -832,7 +853,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
 
 	bkt, err := l.Client.Bucket(destBucket)
 	if err != nil {
@@ -958,7 +979,7 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 		return oi, ossToObjectError(err, bucket, object)
 	}
 
-	return l.GetObjectInfo(ctx, bucket, object)
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy sets policy on bucket.
@@ -1074,3 +1095,8 @@ func (l *ossObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 	}
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *ossObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/s3/gateway-s3.go

@@ -21,11 +21,13 @@ import (
 	"encoding/json"
 	"io"
 	"math/rand"
+	"net/http"
 	"strings"
 	"time"
 
 	"github.com/minio/cli"
 	miniogo "github.com/minio/minio-go"
+	"github.com/minio/minio-go/pkg/credentials"
 	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -69,6 +71,9 @@ ENVIRONMENT VARIABLES:
      MINIO_CACHE_EXPIRY: Cache expiry duration in days.
      MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
 
+  LOGGER:
+     MINIO_LOGGER_HTTP_ENDPOINT: HTTP endpoint URL to log all incoming requests.
+
 EXAMPLES:
   1. Start minio gateway server for AWS S3 backend.
      $ export MINIO_ACCESS_KEY=accesskey
@@ -80,7 +85,13 @@ EXAMPLES:
      $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
      $ {{.HelpName}} https://play.minio.io:9000
 
-  3. Start minio gateway server for AWS S3 backend with edge caching enabled.
+  3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint.
+     $ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
+     $ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     $ export MINIO_LOGGER_HTTP_ENDPOINT="http://localhost:8000/"
+     $ {{.HelpName}} https://play.minio.io:9000
+
+  4. Start minio gateway server for AWS S3 backend with edge caching enabled.
      $ export MINIO_ACCESS_KEY=accesskey
      $ export MINIO_SECRET_KEY=secretkey
      $ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
@@ -88,6 +99,15 @@ EXAMPLES:
      $ export MINIO_CACHE_EXPIRY=40
      $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
+
+  4. Start minio gateway server for AWS S3 backend using AWS environment variables.
+     NOTE: The access and secret key in this case will authenticate with Minio instead
+     of AWS and AWS envs will be used to authenticate to AWS S3.
+     $ export AWS_ACCESS_KEY_ID=aws_access_key
+     $ export AWS_SECRET_ACCESS_KEY=aws_secret_key
+     $ export MINIO_ACCESS_KEY=accesskey
+     $ export MINIO_SECRET_KEY=secretkey
+     $ {{.HelpName}}
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -149,7 +169,7 @@ func randString(n int, src rand.Source, prefix string) string {
 }
 
 // newS3 - Initializes a new client by auto probing S3 server signature.
-func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
+func newS3(url string) (*miniogo.Core, error) {
 	if url == "" {
 		url = "https://s3.amazonaws.com"
 	}
@@ -160,19 +180,36 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 		return nil, err
 	}
 
-	clnt, err := miniogo.NewV4(endpoint, accessKey, secretKey, secure)
+	// Chains all credential types, in the following order:
+	//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+	//  - IAM profile based credentials. (performs an HTTP
+	//    call to a pre-defined endpoint, only valid inside
+	//    configured ec2 instances)
+	//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
+	//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
+	creds := credentials.NewChainCredentials([]credentials.Provider{
+		&credentials.EnvAWS{},
+		&credentials.IAM{
+			Client: &http.Client{
+				Transport: minio.NewCustomHTTPTransport(),
+			},
+		},
+		&credentials.FileAWSCredentials{},
+		&credentials.EnvMinio{},
+	})
+
+	clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "")
 	if err != nil {
 		return nil, err
 	}
+
+	// Set custom transport
+	clnt.SetCustomTransport(minio.NewCustomHTTPTransport())
+
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+	// Check if the provided keys are valid.
 	if _, err = clnt.BucketExists(probeBucketName); err != nil {
-		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
-		if err != nil {
-			return nil, err
-		}
-		if _, err = clnt.BucketExists(probeBucketName); err != nil {
-			return nil, err
-		}
+		return nil, err
 	}
 
 	return &miniogo.Core{Client: clnt}, nil
@@ -180,8 +217,9 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 
 // NewGatewayLayer returns s3 ObjectLayer.
 func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	// Probe S3 signature with input credentials.
-	clnt, err := newS3(g.host, creds.AccessKey, creds.SecretKey)
+	// creds are ignored here, since S3 gateway implements chaining
+	// all credentials.
+	clnt, err := newS3(g.host)
 	if err != nil {
 		return nil, err
 	}
@@ -301,21 +339,47 @@ func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	return minio.FromMinioClientListBucketV2Result(bucket, result), nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, minio.ErrorRespToObjectError(err, bucket, object)
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, minio.ErrorRespToObjectError(err, bucket, object)
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject reads an object from S3. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, o minio.ObjectOptions) error {
 	if length < 0 && length != -1 {
 		return minio.ErrorRespToObjectError(minio.InvalidRange{}, bucket, key)
 	}
 
 	opts := miniogo.GetObjectOptions{}
+	opts.ServerSideEncryption = o.ServerSideEncryption
+
 	if startOffset >= 0 && length >= 0 {
 		if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
-			logger.LogIf(ctx, err)
 			return minio.ErrorRespToObjectError(err, bucket, key)
 		}
 	}
@@ -326,15 +390,14 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 	defer object.Close()
 
 	if _, err := io.Copy(writer, object); err != nil {
-		logger.LogIf(ctx, err)
 		return minio.ErrorRespToObjectError(err, bucket, key)
 	}
 	return nil
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
-func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) {
-	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{})
+func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{miniogo.GetObjectOptions{ServerSideEncryption: opts.ServerSideEncryption}})
 	if err != nil {
 		return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -343,17 +406,20 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
-	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata))
+func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata), opts.ServerSideEncryption)
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
 	}
+	// On success, populate the key & metadata so they are present in the notification
+	oi.Key = object
+	oi.Metadata = minio.ToMinioClientObjectInfoMetadata(metadata)
 
 	return minio.FromMinioClientObjectInfo(bucket, oi), nil
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
-func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	// Set this header such that following CopyObject() always sets the right metadata on the destination.
 	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
 	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
@@ -363,7 +429,7 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
 	if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
 	}
-	return l.GetObjectInfo(ctx, dstBucket, dstObject)
+	return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts)
 }
 
 // DeleteObject deletes a blob in bucket
@@ -387,9 +453,9 @@ func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) {
+func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
 	// Create PutObject options
-	opts := miniogo.PutObjectOptions{UserMetadata: metadata}
+	opts := miniogo.PutObjectOptions{UserMetadata: metadata, ServerSideEncryption: o.ServerSideEncryption}
 	uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
 	if err != nil {
 		return uploadID, minio.ErrorRespToObjectError(err, bucket, object)
@@ -398,8 +464,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 }
 
 // PutObjectPart puts a part of object in bucket
-func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, e error) {
-	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString())
+func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
+	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), opts.ServerSideEncryption)
 	if err != nil {
 		return pi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -410,7 +476,7 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
 
 	srcInfo.UserDefined = map[string]string{
 		"x-amz-copy-source-if-match": srcInfo.ETag,
@@ -429,7 +495,7 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
 func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, e error) {
 	result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
 	if err != nil {
-		return lpi, err
+		return lpi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
 	return minio.FromMinioClientListPartsInfo(result), nil
@@ -448,7 +514,7 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
-	return l.GetObjectInfo(ctx, bucket, object)
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy sets policy on bucket
@@ -485,3 +551,8 @@ func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 	}
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *s3Objects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/s3/gateway-s3_test.go

@@ -50,6 +50,10 @@ func TestS3ToObjectError(t *testing.T) {
 			inputErr:    errResponse("InvalidBucketName"),
 			expectedErr: minio.BucketNameInvalid{},
 		},
+		{
+			inputErr:    errResponse("InvalidPart"),
+			expectedErr: minio.InvalidPart{},
+		},
 		{
 			inputErr:    errResponse("NoSuchBucketPolicy"),
 			expectedErr: minio.BucketPolicyNotFound{},