Ignore migration if config files don't exist (#6448)

* Adds missing information to documentation for credentials and domain environment variables for distributed minio server startup.

.github/ISSUE_TEMPLATE.md

@@ -24,6 +24,10 @@
 <!--- How has this issue affected you? What are you trying to accomplish? -->
 <!--- Providing context helps us come up with a solution that is most useful in the real world -->
 
+## Regression
+<!-- Is this issue a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## Your Environment
 <!--- Include as many relevant details about the environment you experienced the bug in -->
 * Version used (`minio version`):

.github/PULL_REQUEST_TEMPLATE.md

@@ -7,6 +7,10 @@
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->
 
+## Regression
+<!-- Is this PR fixing a regression? (Yes / No) -->
+<!-- If Yes, optionally please include minio version or commit id or PR# that caused this regression, if you have these details. -->
+
 ## How Has This Been Tested?
 <!--- Please describe in detail how you tested your changes. -->
 <!--- Include details of your testing environment, and the tests you ran to -->

.travis.yml

@@ -8,19 +8,26 @@ dist: trusty
 
 language: go
 
+# this ensures PRs based on a local branch are not built twice
+# the downside is that a PR targeting a different branch is not built
+# but as a workaround you can add the branch to this list
+branches:
+  only:
+    - master
+
 matrix:
   include:
     - os: linux
       env:
         - ARCH=x86_64
-      go: 1.10.1
+      go: 1.10.3
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
         - diff -au <(gofmt -s -d pkg) <(printf "")
         - make test GOFLAGS="-timeout 15m -race -v"
+        - make verify
         - make coverage
-        - node --version
         - cd browser && yarn && yarn test && cd ..
 
 before_install:

Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /go/src/github.com/minio/
 COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 
 RUN  \
-     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      apk add --no-cache --virtual .build-deps git && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      go get -v -d github.com/minio/minio && \

Dockerfile.dev

@@ -16,7 +16,7 @@ COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
 COPY . /go/src/github.com/minio/minio
 
 RUN  \
-     apk add --no-cache ca-certificates curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      apk add --no-cache --virtual .build-deps git && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      cd /go/src/github.com/minio/minio && \

Dockerfile.release

@@ -9,8 +9,7 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
 RUN \
-     apk add --no-cache ca-certificates && \
-     apk add --no-cache --virtual .build-deps curl && \
+     apk add --no-cache ca-certificates 'curl>7.61.0' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \

Makefile

@@ -61,6 +61,8 @@ check: test
 test: verifiers build
 	@echo "Running unit tests"
 	@go test $(GOFLAGS) -tags kqueue ./...
+
+verify: build
 	@echo "Verifying build"
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 

appveyor.yml

@@ -12,7 +12,7 @@ clone_folder: c:\gopath\src\github.com\minio\minio
 # Environment variables
 environment:
   GOPATH: c:\gopath
-  GOROOT: c:\go
+  GOROOT: c:\go110
 
 # scripts that run after cloning repository
 install:

browser/app/js/buckets/Policy.js

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from '../constants'
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from '../constants'
 
 import React from "react"
 import { connect } from "react-redux"
@@ -42,37 +42,40 @@ export class Policy extends React.Component {
   render() {
     const {policy, prefix} = this.props
     let newPrefix = prefix
-
     if (newPrefix === '')
       newPrefix = '*'
 
-    return (
-      <div className="pmb-list">
-        <div className="pmbl-item">
-          { newPrefix }
+    if (policy === NONE) {
+      return <noscript />
+    } else {
+      return (
+        <div className="pmb-list">
+          <div className="pmbl-item">
+            { newPrefix }
+          </div>
+          <div className="pmbl-item">
+            <select className="form-control"
+              disabled
+              value={ policy }>
+              <option value={ READ_ONLY }>
+                Read Only
+              </option>
+              <option value={ WRITE_ONLY }>
+                Write Only
+              </option>
+              <option value={ READ_WRITE }>
+                Read and Write
+              </option>
+            </select>
+          </div>
+          <div className="pmbl-item">
+            <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
+              Remove
+            </button>
+          </div>
         </div>
-        <div className="pmbl-item">
-          <select className="form-control"
-            disabled
-            value={ policy }>
-            <option value={ READ_ONLY }>
-              Read Only
-            </option>
-            <option value={ WRITE_ONLY }>
-              Write Only
-            </option>
-            <option value={ READ_WRITE }>
-              Read and Write
-            </option>
-          </select>
-        </div>
-        <div className="pmbl-item">
-          <button className="btn btn-block btn-danger" onClick={ this.removePolicy.bind(this) }>
-            Remove
-          </button>
-        </div>
-      </div>
-    )
+      )
+    }
   }
 }
 

browser/app/js/buckets/__tests__/Policy.test.js

@@ -17,7 +17,7 @@
 import React from "react"
 import { shallow, mount } from "enzyme"
 import { Policy } from "../Policy"
-import { READ_ONLY, WRITE_ONLY, READ_WRITE } from "../../constants"
+import { READ_ONLY, WRITE_ONLY, READ_WRITE, NONE } from "../../constants"
 import web from "../../web"
 
 jest.mock("../../web", () => ({
@@ -31,6 +31,11 @@ describe("Policy", () => {
     shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={READ_ONLY} />)
   })
 
+  it("should not render when policy is listed as 'none'", () => {
+    const wrapper = shallow(<Policy currentBucket={"bucket"} prefix={"foo"} policy={NONE} />)
+    expect(wrapper.find(".pmb-list").length).toBe(0)
+  })
+
   it("should call web.setBucketPolicy and fetchPolicies on submit", () => {
     const fetchPolicies = jest.fn()
     const wrapper = shallow(

browser/app/js/constants.js

@@ -23,6 +23,7 @@ export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
 export const READ_ONLY = "readonly"
 export const WRITE_ONLY = "writeonly"
 export const READ_WRITE = "readwrite"
+export const NONE = "none"
 
 export const SHARE_OBJECT_EXPIRY_DAYS = 5
 export const SHARE_OBJECT_EXPIRY_HOURS = 0

browser/app/js/objects/__tests__/actions.test.js

@@ -19,16 +19,27 @@ import thunk from "redux-thunk"
 import * as actionsObjects from "../actions"
 import * as alertActions from "../../alert/actions"
 import { minioBrowserPrefix } from "../../constants"
+import history from "../../history"
 
 jest.mock("../../web", () => ({
-  LoggedIn: jest.fn(() => true).mockReturnValueOnce(false),
-  ListObjects: jest.fn(() => {
-    return Promise.resolve({
-      objects: [{ name: "test1" }, { name: "test2" }],
-      istruncated: false,
-      nextmarker: "test2",
-      writable: false
-    })
+  LoggedIn: jest
+    .fn(() => true)
+    .mockReturnValueOnce(true)
+    .mockReturnValueOnce(false)
+    .mockReturnValueOnce(false),
+  ListObjects: jest.fn(({ bucketName }) => {
+    if (bucketName === "test-deny") {
+      return Promise.reject({
+        message: "listobjects is denied"
+      })
+    } else {
+      return Promise.resolve({
+        objects: [{ name: "test1" }, { name: "test2" }],
+        istruncated: false,
+        nextmarker: "test2",
+        writable: false
+      })
+    }
   }),
   RemoveObject: jest.fn(({ bucketName, objects }) => {
     if (!bucketName) {
@@ -112,6 +123,9 @@ describe("Objects actions", () => {
       objects: { currentPrefix: "" }
     })
     const expectedActions = [
+      {
+        type: "alert/CLEAR"
+      },
       {
         type: "objects/SET_LIST",
         objects: [{ name: "test1" }, { name: "test2" }],
@@ -143,6 +157,9 @@ describe("Objects actions", () => {
       objects: { currentPrefix: "" }
     })
     const expectedActions = [
+      {
+        type: "alert/CLEAR"
+      },
       {
         type: "objects/APPEND_LIST",
         objects: [{ name: "test1" }, { name: "test2" }],
@@ -160,6 +177,43 @@ describe("Objects actions", () => {
     })
   })
 
+  it("creates objects/RESET_LIST after failing to fetch the objects from bucket with ListObjects denied for LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    const expectedActions = [
+      {
+        type: "alert/CLEAR"
+      },
+      {
+        type: "alert/SET",
+        alert: {
+          type: "danger",
+          message: "listobjects is denied",
+          id: alertActions.alertId
+        }
+      },
+      {
+        type: "object/RESET_LIST"
+      }
+    ]
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      const actions = store.getActions()
+      expect(actions).toEqual(expectedActions)
+    })
+  })
+
+  it("redirect to login after failing to fetch the objects from bucket for non-LoggedIn users", () => {
+    const store = mockStore({
+      buckets: { currentBucket: "test-deny" },
+      objects: { currentPrefix: "" }
+    })
+    return store.dispatch(actionsObjects.fetchObjects()).then(() => {
+      expect(history.location.pathname.endsWith("/login")).toBeTruthy()
+    })
+  })
+
   it("creates objects/SET_SORT_BY and objects/SET_SORT_ORDER when sortObjects is called", () => {
     const store = mockStore({
       objects: {
@@ -198,6 +252,7 @@ describe("Objects actions", () => {
     })
     const expectedActions = [
       { type: "objects/SET_CURRENT_PREFIX", prefix: "abc/" },
+      { type: "alert/CLEAR" },
       { type: "objects/CHECKED_LIST_RESET" }
     ]
     store.dispatch(actionsObjects.selectPrefix("abc/"))
@@ -244,7 +299,11 @@ describe("Objects actions", () => {
     const expectedActions = [
       {
         type: "alert/SET",
-        alert: { type: "danger", message: "Invalid bucket", id: 0 }
+        alert: {
+          type: "danger",
+          message: "Invalid bucket",
+          id: alertActions.alertId
+        }
       }
     ]
     return store.dispatch(actionsObjects.deleteObject("obj1")).then(() => {

browser/app/js/objects/actions.js

@@ -24,9 +24,11 @@ import {
 import { getCurrentBucket } from "../buckets/selectors"
 import { getCurrentPrefix, getCheckedList } from "./selectors"
 import * as alertActions from "../alert/actions"
+import * as bucketActions from "../buckets/actions"
 import { minioBrowserPrefix } from "../constants"
 
 export const SET_LIST = "objects/SET_LIST"
+export const RESET_LIST = "object/RESET_LIST"
 export const APPEND_LIST = "objects/APPEND_LIST"
 export const REMOVE = "objects/REMOVE"
 export const SET_SORT_BY = "objects/SET_SORT_BY"
@@ -45,6 +47,10 @@ export const setList = (objects, marker, isTruncated) => ({
   isTruncated
 })
 
+export const resetList = () => ({
+  type: RESET_LIST
+})
+
 export const appendList = (objects, marker, isTruncated) => ({
   type: APPEND_LIST,
   objects,
@@ -58,6 +64,7 @@ export const fetchObjects = append => {
       buckets: { currentBucket },
       objects: { currentPrefix, marker }
     } = getState()
+    dispatch(alertActions.clear())
     if (currentBucket) {
       return web
       .ListObjects({
@@ -85,8 +92,13 @@ export const fetchObjects = append => {
         dispatch(setPrefixWritable(res.writable))
       })
       .catch(err => {
-        dispatch(alertActions.set({ type: "danger", message: err.message }))
-        history.push("/login")
+        if (web.LoggedIn()) {
+          dispatch(alertActions.set({ type: "danger", message: err.message }))
+          dispatch(resetList())
+        }
+        else {
+          history.push("/login")
+        }
       })
     } 
   }

browser/app/js/objects/reducer.js

@@ -50,6 +50,13 @@ export default (
         marker: action.marker,
         isTruncated: action.isTruncated
       }
+    case actionsObjects.RESET_LIST:
+      return {
+        ...state,
+        list: [],
+        marker: "",
+        isTruncated: false
+      }
     case actionsObjects.APPEND_LIST:
       return {
         ...state,

browser/app/less/inc/list.less

@@ -109,7 +109,15 @@ div.fesl-row {
     /*--------------------------
         Icons
     ----------------------------*/
-    &[data-type=folder] { .list-type(#a1d6dd, '\f114'); }
+    &[data-type=folder] {
+        .list-type(#a1d6dd, '\f114');
+
+        .fesl-item-name {
+            a {
+                cursor: pointer;
+            }
+        }
+    }
     &[data-type=pdf] {.list-type(#fa7775, '\f1c1'); }
     &[data-type=zip] { .list-type(#427089, '\f1c6'); }
     &[data-type=audio] { .list-type(#009688, '\f1c7'); }

cmd/acl-handlers.go

@@ -121,7 +121,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
 	}
 
 	// Before proceeding validate if object exists.
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object)
+	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return

cmd/admin-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,10 +19,13 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"io"
 	"net/http"
+	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -32,6 +35,8 @@ import (
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/quick"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
 )
 
 const (
@@ -460,7 +465,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	configData, err := json.Marshal(config)
+	configData, err := json.MarshalIndent(config, "", "\t")
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
@@ -478,6 +483,90 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	writeSuccessResponseJSON(w, econfigData)
 }
 
+// Disable tidwall json array notation in JSON key path so
+// users can set json with a key as a number.
+// In tidwall json, notify.webhook.0 = val means { "notify" : { "webhook" : [val] }}
+// In Minio, notify.webhook.0 = val means { "notify" : { "webhook" : {"0" : val}}}
+func normalizeJSONKey(input string) (key string) {
+	subKeys := strings.Split(input, ".")
+	for i, k := range subKeys {
+		if i > 0 {
+			key += "."
+		}
+		if _, err := strconv.Atoi(k); err == nil {
+			key += ":" + k
+		} else {
+			key += k
+		}
+	}
+	return
+}
+
+// GetConfigHandler - GET /minio/admin/v1/config-keys
+// Get some keys in config.json of this minio setup.
+func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetConfigKeysHandler")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	var keys []string
+	queries := r.URL.Query()
+
+	for k := range queries {
+		keys = append(keys, k)
+	}
+
+	config, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	configData, err := json.Marshal(config)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	configStr := string(configData)
+	newConfigStr := `{}`
+
+	for _, key := range keys {
+		// sjson.Set does not return an error if key is empty
+		// we should check by ourselves here
+		if key == "" {
+			continue
+		}
+		val := gjson.Get(configStr, key)
+		if j, err := sjson.Set(newConfigStr, normalizeJSONKey(key), val.Value()); err == nil {
+			newConfigStr = j
+		}
+	}
+
+	password := config.GetCredential().SecretKey
+	econfigData, err := madmin.EncryptServerConfigData(password, []byte(newConfigStr))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, []byte(econfigData))
+}
+
 // toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
 // specific error.
 func toAdminAPIErrCode(err error) APIErrorCode {
@@ -507,6 +596,12 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
 	// Read configuration bytes from request body.
 	configBuf := make([]byte, maxConfigJSONSize+1)
 	n, err := io.ReadFull(r.Body, configBuf)
@@ -561,7 +656,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err = saveServerConfig(objectAPI, &config); err != nil {
+	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
 	}
@@ -572,6 +667,139 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	sendServiceCmd(globalAdminPeers, serviceRestart)
 }
 
+func convertValueType(elem []byte, jsonType gjson.Type) (interface{}, error) {
+	str := string(elem)
+	switch jsonType {
+	case gjson.False, gjson.True:
+		return strconv.ParseBool(str)
+	case gjson.JSON:
+		return gjson.Parse(str).Value(), nil
+	case gjson.String:
+		return str, nil
+	case gjson.Number:
+		return strconv.ParseFloat(str, 64)
+	default:
+		return nil, nil
+	}
+}
+
+// SetConfigKeysHandler - PUT /minio/admin/v1/config-keys
+func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SetConfigKeysHandler")
+
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Deny if WORM is enabled
+	if globalWORMEnabled {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	// Validate request signature.
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	// Load config
+	configStruct, err := readServerConfig(ctx, objectAPI)
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	// Convert config to json bytes
+	configBytes, err := json.Marshal(configStruct)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	configStr := string(configBytes)
+
+	queries := r.URL.Query()
+	password := globalServerConfig.GetCredential().SecretKey
+
+	// Set key values in the JSON config
+	for k := range queries {
+		// Decode encrypted data associated to the current key
+		encryptedElem, dErr := base64.StdEncoding.DecodeString(queries.Get(k))
+		if dErr != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("key", k)
+			ctx = logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, dErr)
+			writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
+			return
+		}
+		elem, dErr := madmin.DecryptServerConfigData(password, bytes.NewBuffer([]byte(encryptedElem)))
+		if dErr != nil {
+			logger.LogIf(ctx, dErr)
+			writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
+			return
+		}
+		// Calculate the type of the current key from the
+		// original config json
+		jsonFieldType := gjson.Get(configStr, k).Type
+		// Convert passed value to json filed type
+		val, cErr := convertValueType(elem, jsonFieldType)
+		if cErr != nil {
+			writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, cErr.Error(), r.URL)
+			return
+		}
+		// Set the key/value in the new json document
+		if s, sErr := sjson.Set(configStr, normalizeJSONKey(k), val); sErr == nil {
+			configStr = s
+		}
+	}
+
+	configBytes = []byte(configStr)
+
+	// Validate config
+	var config serverConfig
+	if err = json.Unmarshal(configBytes, &config); err != nil {
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
+		return
+	}
+
+	if err = config.Validate(); err != nil {
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
+		return
+	}
+
+	if err = config.TestNotificationTargets(); err != nil {
+		writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
+		return
+	}
+
+	// If credentials for the server are provided via environment,
+	// then credentials in the provided configuration must match.
+	if globalIsEnvCreds {
+		creds := globalServerConfig.GetCredential()
+		if config.Credential.AccessKey != creds.AccessKey ||
+			config.Credential.SecretKey != creds.SecretKey {
+			writeErrorResponseJSON(w, ErrAdminCredentialsMismatch, r.URL)
+			return
+		}
+	}
+
+	if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
+		return
+	}
+
+	// Send success response
+	writeSuccessResponseHeadersOnly(w)
+
+	sendServiceCmd(globalAdminPeers, serviceRestart)
+}
+
 // UpdateCredsHandler - POST /minio/admin/v1/config/credential
 // ----------
 // Update credentials in a minio server. In a distributed setup,
@@ -645,16 +873,17 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
 	// Update local credentials in memory.
 	globalServerConfig.SetCredential(creds)
 
-	if err = saveServerConfig(objectAPI, globalServerConfig); err != nil {
+	if err = saveServerConfig(ctx, objectAPI, globalServerConfig); err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
 		return
 	}
 
 	// Notify all other Minio peers to update credentials
 	for host, err := range globalNotificationSys.LoadCredentials() {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", host.String())
-		ctx := logger.SetReqInfo(ctx, reqInfo)
-		logger.LogIf(ctx, err)
+		if err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
+			logger.LogIf(ctx, err)
+		}
 	}
 
 	// Reply to the client before restarting minio server.

cmd/admin-handlers_test.go

@@ -101,7 +101,17 @@ var (
 			"1": {
 				"enable": false,
 				"brokers": null,
-				"topic": ""
+				"topic": "",
+				"tls" : {
+					"enable" : false,
+					"skipVerify" : false,
+					"clientAuth" : 0
+				},
+				"sasl" : {
+					"enable" : false,
+					"username" : "",
+					"password" : ""
+				}
 			}
 		},
 		"mqtt": {
@@ -274,7 +284,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 			objectName := fmt.Sprintf("%s-%d", objName, i)
 			_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
 				mustGetHashReader(t, bytes.NewReader([]byte("hello")),
-					int64(len("hello")), "", ""), nil)
+					int64(len("hello")), "", ""), nil, ObjectOptions{})
 			if err != nil {
 				t.Fatalf("Failed to create %s - %v", objectName,
 					err)
@@ -286,14 +296,14 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
 	{
 		objName := "mpObject"
 		uploadID, err := atb.objLayer.NewMultipartUpload(context.Background(), bucketName,
-			objName, nil)
+			objName, nil, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp new error: %v", err)
 		}
 
 		_, err = atb.objLayer.PutObjectPart(context.Background(), bucketName, objName,
 			uploadID, 3, mustGetHashReader(t, bytes.NewReader(
-				[]byte("hello")), int64(len("hello")), "", ""))
+				[]byte("hello")), int64(len("hello")), "", ""), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("mp put error: %v", err)
 		}

cmd/admin-heal-ops.go

@@ -610,7 +610,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		if h.objPrefix != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix)
+			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
 			if err == nil {
 				err = h.healObject(bucket, h.objPrefix)
 				if err != nil {

cmd/admin-router.go

@@ -68,4 +68,9 @@ func registerAdminRouter(router *mux.Router) {
 	adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
 	// Set config
 	adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+
+	// Get config keys/values
+	adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
+	// Set config keys/values
+	adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
 }

cmd/api-errors.go

@@ -129,6 +129,7 @@ const (
 	ErrMaximumExpires
 	ErrSlowDown
 	ErrInvalidPrefixMarker
+	ErrBadRequest
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -289,6 +290,7 @@ const (
 	ErrEvaluatorBindingDoesNotExist
 	ErrInvalidColumnIndex
 	ErrMissingHeaders
+	ErrAdminConfigNotificationTargetsFailed
 )
 
 // error code to APIError structure, these fields carry respective
@@ -636,6 +638,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Invalid marker prefix combination",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrBadRequest: {
+		Code:           "BadRequest",
+		Description:    "400 BadRequest",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
 	ErrUnsignedHeaders: {
@@ -880,6 +887,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "JSON configuration provided has objects with duplicate keys",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminConfigNotificationTargetsFailed: {
+		Code:           "XMinioAdminNotificationTargetsTestFailed",
+		Description:    "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminCredentialsMismatch: {
 		Code:           "XMinioAdminCredentialsMismatch",
 		Description:    "Credentials in config mismatch with server environment variables",
@@ -1429,7 +1441,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrSSEEncryptedObject
 	case errInvalidSSEParameters:
 		apiErr = ErrInvalidSSECustomerParameters
-	case crypto.ErrInvalidCustomerKey:
+	case crypto.ErrInvalidCustomerKey, crypto.ErrSecretKeyMismatch:
 		apiErr = ErrAccessDenied // no access without correct key
 	case crypto.ErrIncompatibleEncryptionMethod:
 		apiErr = ErrIncompatibleEncryptionMethod
@@ -1437,7 +1449,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case crypto.ErrKMSAuthLogin:
 		apiErr = ErrKMSAuthFailure
-	case context.Canceled, context.DeadlineExceeded:
+	case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	}
 	switch err {

cmd/benchmark-utils_test.go

@@ -61,7 +61,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	for i := 0; i < b.N; i++ {
 		// insert the object.
 		objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -97,7 +97,8 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	// generate md5sum for the generated data.
 	// md5sum of the data to written is required as input for NewMultipartUpload.
 	metadata := make(map[string]string)
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata)
+	opts := ObjectOptions{}
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucket, object, metadata, opts)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -122,7 +123,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 			md5hex = getMD5Hash([]byte(textPartData))
 			var partInfo PartInfo
 			partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
-				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
+				mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -203,7 +204,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -218,7 +219,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		var buffer = new(bytes.Buffer)
-		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "")
+		err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "", ObjectOptions{})
 		if err != nil {
 			b.Error(err)
 		}
@@ -303,7 +304,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		for pb.Next() {
 			// insert the object.
 			objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+				mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -343,7 +344,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 		// insert the object.
 		var objInfo ObjectInfo
 		objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
-			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
+			mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}
@@ -359,7 +360,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	b.RunParallel(func(pb *testing.PB) {
 		i := 0
 		for pb.Next() {
-			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "")
+			err = obj.GetObject(context.Background(), bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "", ObjectOptions{})
 			if err != nil {
 				b.Error(err)
 			}

cmd/bitrot.go

@@ -23,7 +23,7 @@ import (
 
 	"github.com/minio/highwayhash"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/sha256-simd"
+	sha256 "github.com/minio/sha256-simd"
 	"golang.org/x/crypto/blake2b"
 )
 
@@ -70,7 +70,7 @@ func (a BitrotAlgorithm) New() hash.Hash {
 	}
 }
 
-// Available reports whether the given algorihm is available.
+// Available reports whether the given algorithm is available.
 func (a BitrotAlgorithm) Available() bool {
 	_, ok := bitrotAlgorithms[a]
 	return ok

cmd/bucket-handlers.go

@@ -622,7 +622,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		}
 	}
 
-	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata)
+	objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata, ObjectOptions{})
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return

cmd/bucket-handlers_test.go

@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
+		_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object %d:  Error uploading object: <ERROR> %v", i, err)

cmd/common-main.go

@@ -114,11 +114,6 @@ func handleCommonEnvVars() {
 		globalActiveCred = cred
 	}
 
-	// In distributed setup users need to set ENVs always.
-	if !globalIsEnvCreds && globalIsDistXL {
-		logger.Fatal(uiErrEnvCredentialsMissingServer(nil), "Unable to start distributed server mode")
-	}
-
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
 		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {

cmd/config-current.go

@@ -64,6 +64,12 @@ func (s *serverConfig) SetRegion(region string) {
 
 // GetRegion get current region.
 func (s *serverConfig) GetRegion() string {
+	if globalIsEnvRegion {
+		return globalServerRegion
+	}
+	if s == nil {
+		return ""
+	}
 	return s.Region
 }
 
@@ -111,16 +117,34 @@ func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
 // GetStorageClass reads storage class fields from current config.
 // It returns the standard and reduced redundancy storage class struct
 func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
+	if globalIsStorageClass {
+		return globalStandardStorageClass, globalRRStorageClass
+	}
+	if s == nil {
+		return storageClass{}, storageClass{}
+	}
 	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
 // GetBrowser get current credentials.
 func (s *serverConfig) GetBrowser() bool {
+	if globalIsEnvBrowser {
+		return globalIsBrowserEnabled
+	}
+	if s == nil {
+		return true
+	}
 	return bool(s.Browser)
 }
 
 // GetWorm get current credentials.
 func (s *serverConfig) GetWorm() bool {
+	if globalIsEnvWORM {
+		return globalWORMEnabled
+	}
+	if s == nil {
+		return false
+	}
 	return bool(s.Worm)
 }
 
@@ -134,10 +158,24 @@ func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxu
 
 // GetCacheConfig gets the current cache config
 func (s *serverConfig) GetCacheConfig() CacheConfig {
+	if globalIsDiskCacheEnabled {
+		return CacheConfig{
+			Drives:  globalCacheDrives,
+			Exclude: globalCacheExcludes,
+			Expiry:  globalCacheExpiry,
+			MaxUse:  globalCacheMaxUse,
+		}
+	}
+	if s == nil {
+		return CacheConfig{}
+	}
 	return s.Cache
 }
 
 func (s *serverConfig) Validate() error {
+	if s == nil {
+		return nil
+	}
 	if s.Version != serverConfigVersion {
 		return fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, s.Version)
 	}
@@ -250,6 +288,102 @@ func (s *serverConfig) loadFromEnvs() {
 	}
 }
 
+// TestNotificationTargets tries to establish connections to all notification
+// targets when enabled. This is a good way to make sure all configurations
+// set by the user can work.
+func (s *serverConfig) TestNotificationTargets() error {
+	for k, v := range s.Notify.AMQP {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewAMQPTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("amqp(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Elasticsearch {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewElasticsearchTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Kafka {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewKafkaTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("kafka(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MQTT {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMQTTTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mqtt(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.MySQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewMySQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("mysql(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.NATS {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewNATSTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("nats(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.PostgreSQL {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewPostgreSQLTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("postgreSQL(%s): %s", k, err.Error())
+		}
+		t.Close()
+	}
+
+	for k, v := range s.Notify.Redis {
+		if !v.Enable {
+			continue
+		}
+		t, err := target.NewRedisTarget(k, v)
+		if err != nil {
+			return fmt.Errorf("redis(%s): %s", k, err.Error())
+		}
+		t.Close()
+
+	}
+
+	return nil
+}
+
 // Returns the string describing a difference with the given
 // configuration object. If the given configuration object is
 // identical, an empty string is returned.
@@ -410,7 +544,7 @@ func newConfig(objAPI ObjectLayer) error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return saveServerConfig(objAPI, globalServerConfig)
+	return saveServerConfig(context.Background(), objAPI, globalServerConfig)
 }
 
 // getValidConfig - returns valid server configuration
@@ -452,7 +586,9 @@ func loadConfig(objAPI ObjectLayer) error {
 // * Handle the configuration in this function to create/add into TargetList.
 func getNotificationTargets(config *serverConfig) *event.TargetList {
 	targetList := event.NewTargetList()
-
+	if config == nil {
+		return targetList
+	}
 	for id, args := range config.Notify.AMQP {
 		if args.Enable {
 			newTarget, err := target.NewAMQPTarget(id, args)

cmd/config-current_test.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"os"
 	"path"
 	"testing"
@@ -51,7 +52,7 @@ func TestServerConfig(t *testing.T) {
 		t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
 	}
 
-	if err := saveServerConfig(objLayer, globalServerConfig); err != nil {
+	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {
 		t.Fatalf("Unable to save updated config file %s", err)
 	}
 
@@ -66,6 +67,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	os.Setenv("MINIO_BROWSER", "off")
 	defer os.Unsetenv("MINIO_BROWSER")
 
+	os.Setenv("MINIO_WORM", "on")
+	defer os.Unsetenv("MINIO_WORM")
+
 	os.Setenv("MINIO_ACCESS_KEY", "minio")
 	defer os.Unsetenv("MINIO_ACCESS_KEY")
 
@@ -97,31 +101,37 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	serverHandleEnvVars()
 
 	// Init config
-	initConfig()
+	initConfig(objLayer)
 
-	// Check if serverConfig has
+	// Check if serverConfig has browser disabled
 	if globalServerConfig.GetBrowser() {
-		t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
+		t.Error("Expected browser to be disabled but it is not")
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig returns WORM config from the env
+	if !globalServerConfig.GetWorm() {
+		t.Error("Expected WORM to be enabled but it is not")
+	}
+
+	// Check if serverConfig has region from the environment
 	if globalServerConfig.GetRegion() != "us-west-1" {
-		t.Errorf("Expecting region to be \"us-west-1\" found %v", globalServerConfig.GetRegion())
+		t.Errorf("Expected region to be \"us-west-1\", found %v", globalServerConfig.GetRegion())
 	}
 
-	// Check if serverConfig has
+	// Check if serverConfig has credentials from the environment
 	cred := globalServerConfig.GetCredential()
 
 	if cred.AccessKey != "minio" {
-		t.Errorf("Expecting access key to be `minio` found %s", cred.AccessKey)
+		t.Errorf("Expected access key to be `minio`, found %s", cred.AccessKey)
 	}
 
 	if cred.SecretKey != "minio123" {
-		t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
+		t.Errorf("Expected access key to be `minio123`, found %s", cred.SecretKey)
 	}
 
+	// Check if serverConfig has the correct domain
 	if globalServerConfig.Domain != "domain.com" {
-		t.Errorf("Expecting Domain to be `domain.com` found " + globalServerConfig.Domain)
+		t.Errorf("Expected Domain to be `domain.com`, found " + globalServerConfig.Domain)
 	}
 }
 

cmd/config-migrate.go

@@ -2411,18 +2411,6 @@ func migrateV27ToV28() error {
 
 // Migrates '.minio.sys/config.json' v27 to v28.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
-	// Construct path to config.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, minioConfigFile)
-	transactionConfigFile := configFile + ".transaction"
-
-	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
-	// and configFile, take a transaction lock to avoid race.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
-
 	return migrateV27ToV28MinioSys(objAPI)
 }
 
@@ -2440,7 +2428,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 
 	srvConfig.Version = "28"
 	srvConfig.KMS = crypto.KMSConfig{}
-	if err = saveServerConfig(objAPI, srvConfig); err != nil {
+	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
 	}
 

cmd/config-migrate_test.go

@@ -174,10 +174,6 @@ func TestServerConfigMigrateV2toV28(t *testing.T) {
 	}
 	defer os.RemoveAll(fsDir)
 
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
 	configPath := rootPath + "/" + minioConfigFile
 
 	// Create a corrupted config file
@@ -203,12 +199,12 @@ func TestServerConfigMigrateV2toV28(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
-	if err := migrateConfigToMinioSys(); err != nil {
+	if err := migrateConfigToMinioSys(objLayer); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(newObjectLayerFn()); err != nil {
+	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 

cmd/config.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"io"
 	"io/ioutil"
+	"os"
 	"path"
 	"runtime"
 	"time"
@@ -37,26 +38,49 @@ const (
 
 	// Minio configuration file.
 	minioConfigFile = "config.json"
+
+	// Minio backup file
+	minioConfigBackupFile = minioConfigFile + ".backup"
 )
 
-func saveServerConfig(objAPI ObjectLayer, config *serverConfig) error {
+func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverConfig) error {
 	if err := quick.CheckData(config); err != nil {
 		return err
 	}
 
-	data, err := json.Marshal(config)
+	data, err := json.MarshalIndent(config, "", "\t")
 	if err != nil {
 		return err
 	}
 
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	if globalEtcdClient != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
-		_, err := globalEtcdClient.Put(ctx, configFile, string(data))
+		timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+		_, err = globalEtcdClient.Put(timeoutCtx, configFile, string(data))
 		defer cancel()
 		return err
 	}
 
+	// Create a backup of the current config
+	reader, err := readConfig(ctx, objAPI, configFile)
+	if err == nil {
+		var oldData []byte
+		oldData, err = ioutil.ReadAll(reader)
+		if err != nil {
+			return err
+		}
+		backupConfigFile := path.Join(minioConfigPrefix, minioConfigBackupFile)
+		err = saveConfig(objAPI, backupConfigFile, oldData)
+		if err != nil {
+			return err
+		}
+	} else {
+		if err != errConfigNotFound {
+			return err
+		}
+	}
+
+	// Save the new config in the std config path
 	return saveConfig(objAPI, configFile, data)
 }
 
@@ -135,8 +159,9 @@ func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
 		return checkServerConfigEtcd(configFile)
 	}
 
-	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile); err != nil {
-		if isErrObjectNotFound(err) {
+	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
+		// Convert ObjectNotFound, Quorum errors into errConfigNotFound
+		if isErrObjectNotFound(err) || isInsufficientReadQuorum(err) {
 			return errConfigNotFound
 		}
 		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
@@ -152,7 +177,7 @@ func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
 		return err
 	}
 
-	_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil)
+	_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
 	return err
 }
 
@@ -161,8 +186,8 @@ var errConfigNotFound = errors.New("config file not found")
 func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
 	var buffer bytes.Buffer
 	// Read entire content by setting size to -1
-	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, ""); err != nil {
-		// Ignore if err is ObjectNotFound or IncompleteBody when bucket is not configured with notification
+	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+		// Convert ObjectNotFound, IncompleteBody and Quorum errors into errConfigNotFound
 		if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
 			return nil, errConfigNotFound
 		}
@@ -193,7 +218,7 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errInvalidArgument
 	}
-	return loadConfig(objAPI)
+	return initConfig(objAPI)
 }
 
 // NewConfigSys - creates new config system object.
@@ -201,67 +226,70 @@ func NewConfigSys() *ConfigSys {
 	return &ConfigSys{}
 }
 
-// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/minio.json'
-func migrateConfigToMinioSys() error {
-	// Construct path to config.json for the given bucket.
-	configFile := path.Join(bucketConfigPrefix, minioConfigFile)
-	transactionConfigFile := configFile + ".transaction"
-
-	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
-	// and configFile, take a transaction lock to avoid race.
-	objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
-	if err := objLock.GetLock(globalOperationTimeout); err != nil {
-		return err
-	}
-	defer objLock.Unlock()
+// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
+func migrateConfigToMinioSys(objAPI ObjectLayer) error {
+	defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
 
 	// Verify if backend already has the file.
-	if err := checkServerConfig(context.Background(), newObjectLayerFn()); err != errConfigNotFound {
+	if err := checkServerConfig(context.Background(), objAPI); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
 
 	var config = &serverConfig{}
 	if _, err := Load(getConfigFile(), config); err != nil {
-		return err
+		if !os.IsNotExist(err) {
+			return err
+		}
+		// Read from deprecate file as well if necessary.
+		if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
+			return err
+		}
 	}
 
-	return saveServerConfig(newObjectLayerFn(), config)
+	return saveServerConfig(context.Background(), objAPI, config)
 }
 
 // Initialize and load config from remote etcd or local config directory
-func initConfig() {
+func initConfig(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	if globalEtcdClient != nil {
 		ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
 		resp, err := globalEtcdClient.Get(ctx, getConfigFile())
 		cancel()
 		if err == nil && resp.Count > 0 {
-			logger.FatalIf(migrateConfig(), "Config migration failed")
+			return migrateConfig()
 		}
 	} else {
 		if isFile(getConfigFile()) {
-			logger.FatalIf(migrateConfig(), "Config migration failed")
-
-			// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
-			if err := migrateConfigToMinioSys(); err != nil {
-				logger.Fatal(err, "Unable to migrate 'config.json' to '.minio.sys/config/config.json'")
+			if err := migrateConfig(); err != nil {
+				return err
 			}
 		}
+		// Migrates ${HOME}/.minio/config.json or config.json.deprecated
+		// to '<export_path>/.minio.sys/config/config.json'
+		// ignore if the file doesn't exist.
+		if err := migrateConfigToMinioSys(objAPI); err != nil && !os.IsNotExist(err) {
+			return err
+		}
 	}
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		logger.FatalIf(errServerNotInitialized, "Server is not initialized yet unable to proceed")
-	}
+
 	if err := checkServerConfig(context.Background(), objAPI); err != nil {
 		if err == errConfigNotFound {
 			// Config file does not exist, we create it fresh and return upon success.
-			logger.FatalIf(newConfig(objAPI), "Unable to initialize minio config for the first time")
-			logger.Info("Created minio configuration file successfully at " + getConfigDir())
+			if err = newConfig(objAPI); err != nil {
+				return err
+			}
 		} else {
-			logger.FatalIf(err, "Unable to load the configuration file")
+			return err
 		}
 	}
 
-	logger.FatalIf(migrateMinioSysConfig(objAPI), "Config migration failed for minio.sys config")
+	if err := migrateMinioSysConfig(objAPI); err != nil {
+		return err
+	}
 
-	logger.FatalIf(loadConfig(objAPI), "Unable to load the configuration file")
+	return loadConfig(objAPI)
 }

cmd/crypto/error.go

@@ -43,6 +43,10 @@ var (
 	// base64-encoded string or not 256 bits long.
 	ErrInvalidCustomerKey = errors.New("The SSE-C client key is invalid")
 
+	// ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key)
+	// does not match the secret key used during encrypting the object.
+	ErrSecretKeyMismatch = errors.New("The secret key does not match the secret key used during upload")
+
 	// ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the
 	// computed MD5 sum. This means that the client provided either the wrong key for
 	// a certain MD5 checksum or the wrong MD5 for a certain key.

cmd/crypto/key.go

@@ -124,7 +124,7 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 	}
 
 	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
-		return err // TODO(aead): upgrade sio to use sio.Error
+		return ErrSecretKeyMismatch
 	}
 	copy(key[:], decryptedKey.Bytes())
 	return nil

cmd/disk-cache-fs.go

@@ -258,7 +258,7 @@ func (cfs *cacheFSObjects) IsOnline() bool {
 }
 
 // Caches the object to disk
-func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) error {
+func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) error {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:
@@ -275,7 +275,7 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 			return pErr
 		}
 	}
-	_, err := cfs.PutObject(ctx, bucket, object, data, metadata)
+	_, err := cfs.PutObject(ctx, bucket, object, data, metadata, opts)
 	// if err is due to disk being offline , mark cache drive as offline
 	if IsErr(err, baseErrs...) {
 		cfs.setOnline(false)
@@ -284,8 +284,8 @@ func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data
 }
 
 // Returns the handle for the cached object
-func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
-	return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag)
+func (cfs *cacheFSObjects) Get(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
+	return cfs.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
 }
 
 // Deletes the cached object
@@ -295,13 +295,13 @@ func (cfs *cacheFSObjects) Delete(ctx context.Context, bucket, object string) (e
 
 // convenience function to check if object is cached on this cacheFSObjects
 func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bool {
-	_, err := cfs.GetObjectInfo(ctx, bucket, object)
+	_, err := cfs.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
 	return err == nil
 }
 
 // Identical to fs PutObject operation except that it uses ETag in metadata
 // headers.
-func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	fs := cfs.FSObjects
 	// Lock the object.
 	objectLock := fs.nsMutex.NewNSLock(bucket, object)
@@ -438,7 +438,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 // Implements S3 compatible initiate multipart API. Operation here is identical
 // to fs backend implementation - with the exception that cache FS uses the uploadID
 // generated on the backend
-func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string) (string, error) {
+func (cfs *cacheFSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, uploadID string, opts ObjectOptions) (string, error) {
 	if cfs.diskUsageHigh() {
 		select {
 		case cfs.purgeChan <- struct{}{}:

cmd/disk-cache.go

@@ -57,16 +57,16 @@ type cacheObjects struct {
 	// file path patterns to exclude from cache
 	exclude []string
 	// Object functions pointing to the corresponding functions of backend implementation.
-	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
-	GetObjectInfoFn           func(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error)
-	PutObjectFn               func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
+	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObjectFn               func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObjectFn            func(ctx context.Context, bucket, object string) error
 	ListObjectsFn             func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2Fn           func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListBucketsFn             func(ctx context.Context) (buckets []BucketInfo, err error)
 	GetBucketInfoFn           func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
-	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error)
-	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
+	NewMultipartUploadFn      func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	PutObjectPartFn           func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUploadFn    func(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
 	DeleteBucketFn            func(ctx context.Context, bucket string) error
@@ -88,14 +88,14 @@ type CacheObjectLayer interface {
 	ListBuckets(ctx context.Context) (buckets []BucketInfo, err error)
 	DeleteBucket(ctx context.Context, bucket string) error
 	// Object operations.
-	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
-	GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
+	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
-	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error)
-	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
+	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
 
@@ -177,20 +177,20 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string {
 
 // Uses cached-object to serve the request. If object is not cached it serves the request from the backend and also
 // stores it in the cache for serving subsequent requests.
-func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	GetObjectFn := c.GetObjectFn
 	GetObjectInfoFn := c.GetObjectInfoFn
 
 	if c.isCacheExclude(bucket, object) {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
 	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
 	if err != nil {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// stat object on backend
-	objInfo, err := GetObjectInfoFn(ctx, bucket, object)
+	objInfo, err := GetObjectInfoFn(ctx, bucket, object, opts)
 	backendDown := backendDownError(err)
 	if err != nil && !backendDown {
 		if _, ok := err.(ObjectNotFound); ok {
@@ -201,27 +201,27 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 	}
 
 	if !backendDown && filterFromCache(objInfo.UserDefined) {
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 
-	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object)
+	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts)
 	if err == nil {
 		if backendDown {
 			// If the backend is down, serve the request from cache.
-			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag)
+			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		}
 		if cachedObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
-			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag)
+			return dcache.Get(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		}
 		dcache.Delete(ctx, bucket, object)
 	}
 	if startOffset != 0 || length != objInfo.Size {
 		// We don't cache partial objects.
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
 		// cache only objects < 1/100th of disk capacity
-		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag)
+		return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
 	}
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
@@ -230,13 +230,13 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 		return err
 	}
 	go func() {
-		if err = GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag); err != nil {
+		if err = GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts); err != nil {
 			pipeWriter.CloseWithError(err)
 			return
 		}
 		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
 	}()
-	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
+	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo), opts)
 	if err != nil {
 		return err
 	}
@@ -245,17 +245,17 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 }
 
 // Returns ObjectInfo from cache if available.
-func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string) (ObjectInfo, error) {
+func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 	getObjectInfoFn := c.GetObjectInfoFn
 	if c.isCacheExclude(bucket, object) {
-		return getObjectInfoFn(ctx, bucket, object)
+		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
 	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
 	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
 	if err != nil {
-		return getObjectInfoFn(ctx, bucket, object)
+		return getObjectInfoFn(ctx, bucket, object, opts)
 	}
-	objInfo, err := getObjectInfoFn(ctx, bucket, object)
+	objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
 	if err != nil {
 		if _, ok := err.(ObjectNotFound); ok {
 			// Delete the cached entry if backend object was deleted.
@@ -266,14 +266,14 @@ func (c cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 			return ObjectInfo{}, err
 		}
 		// when backend is down, serve from cache.
-		cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object)
+		cachedObjInfo, cerr := dcache.GetObjectInfo(ctx, bucket, object, opts)
 		if cerr == nil {
 			return cachedObjInfo, nil
 		}
 		return ObjectInfo{}, BackendDown{}
 	}
 	// when backend is up, do a sanity check on cached object
-	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object)
+	cachedObjInfo, err := dcache.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		return objInfo, nil
 	}
@@ -560,24 +560,24 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool {
 }
 
 // PutObject - caches the uploaded object for single Put operations
-func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	putObjectFn := c.PutObjectFn
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectFn(ctx, bucket, object, r, metadata)
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	size := r.Size()
 
 	// fetch from backend if there is no space on cache drive
 	if !dcache.diskAvailable(size * cacheSizeMultiplier) {
-		return putObjectFn(ctx, bucket, object, r, metadata)
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	// fetch from backend if cache exclude pattern or cache-control
 	// directive set to exclude
 	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
 		dcache.Delete(ctx, bucket, object)
-		return putObjectFn(ctx, bucket, object, r, metadata)
+		return putObjectFn(ctx, bucket, object, r, metadata, opts)
 	}
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
@@ -595,7 +595,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	oinfoCh := make(chan ObjectInfo)
 	errCh := make(chan error)
 	go func() {
-		oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata)
+		oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata, opts)
 		if perr != nil {
 			pipeWriter.CloseWithError(perr)
 			wPipe.CloseWithError(perr)
@@ -608,7 +608,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	}()
 
 	go func() {
-		if err = dcache.Put(ctx, bucket, object, cHashReader, metadata); err != nil {
+		if err = dcache.Put(ctx, bucket, object, cHashReader, metadata, opts); err != nil {
 			wPipe.CloseWithError(err)
 			return
 		}
@@ -627,39 +627,39 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 }
 
 // NewMultipartUpload - Starts a new multipart upload operation to backend and cache.
-func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	newMultipartUploadFn := c.NewMultipartUploadFn
 
 	if c.isCacheExclude(bucket, object) || filterFromCache(metadata) {
-		return newMultipartUploadFn(ctx, bucket, object, metadata)
+		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	}
 
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return newMultipartUploadFn(ctx, bucket, object, metadata)
+		return newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	}
 
-	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata)
+	uploadID, err = newMultipartUploadFn(ctx, bucket, object, metadata, opts)
 	if err != nil {
 		return
 	}
 	// create new multipart upload in cache with same uploadID
-	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID)
+	dcache.NewMultipartUpload(ctx, bucket, object, metadata, uploadID, opts)
 	return uploadID, err
 }
 
 // PutObjectPart - uploads part to backend and cache simultaneously.
-func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
 	putObjectPartFn := c.PutObjectPartFn
 	dcache, err := c.cache.getCacheFS(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
 	if c.isCacheExclude(bucket, object) {
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
 	// make sure cache has at least cacheSizeMultiplier * size available
@@ -669,7 +669,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		case dcache.purgeChan <- struct{}{}:
 		default:
 		}
-		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data)
+		return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
 	}
 
 	info = PartInfo{}
@@ -688,7 +688,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 	pinfoCh := make(chan PartInfo)
 	errorCh := make(chan error)
 	go func() {
-		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader)
+		info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader, opts)
 		if err != nil {
 			close(pinfoCh)
 			pipeWriter.CloseWithError(err)
@@ -700,7 +700,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 		pinfoCh <- info
 	}()
 	go func() {
-		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader); perr != nil {
+		if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader, opts); perr != nil {
 			wPipe.CloseWithError(perr)
 			return
 		}
@@ -876,14 +876,14 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		cache:    dcache,
 		exclude:  config.Exclude,
 		listPool: newTreeWalkPool(globalLookupTimeout),
-		GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
-			return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag)
+		GetObjectFn: func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
+			return newObjectLayerFn().GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
 		},
-		GetObjectInfoFn: func(ctx context.Context, bucket, object string) (ObjectInfo, error) {
-			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object)
+		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
-			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata)
+		PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+			return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
 		},
 		DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object)
@@ -900,11 +900,11 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
 		GetBucketInfoFn: func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error) {
 			return newObjectLayerFn().GetBucketInfo(ctx, bucket)
 		},
-		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
-			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata)
+		NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
+			return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts)
 		},
-		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
-			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data)
+		PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
+			return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
 		},
 		AbortMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string) error {
 			return newObjectLayerFn().AbortMultipartUpload(ctx, bucket, object, uploadID)

cmd/disk-cache_test.go

@@ -192,17 +192,18 @@ func TestDiskCache(t *testing.T) {
 	objInfo.ContentType = contentType
 	objInfo.ETag = etag
 	objInfo.UserDefined = httpMeta
+	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
 	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
 	if err != nil {
 		t.Fatal(err)
 	}
-	err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+	err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
-	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+	cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -219,7 +220,7 @@ func TestDiskCache(t *testing.T) {
 		t.Fatal("Cached content-type does not match")
 	}
 	writer := bytes.NewBuffer(nil)
-	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+	err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -266,6 +267,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	objInfo.ContentType = contentType
 	objInfo.ETag = etag
 	objInfo.UserDefined = httpMeta
+	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
 	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
@@ -273,16 +275,16 @@ func TestDiskCacheMaxUse(t *testing.T) {
 		t.Fatal(err)
 	}
 	if !cache.diskAvailable(int64(size)) {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
 		if err != errDiskFull {
 			t.Fatal("Cache max-use limit violated.")
 		}
 	} else {
-		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
+		err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
 		if err != nil {
 			t.Fatal(err)
 		}
-		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
+		cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName, opts)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -299,7 +301,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 			t.Fatal("Cached content-type does not match")
 		}
 		writer := bytes.NewBuffer(nil)
-		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
+		err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "", opts)
 		if err != nil {
 			t.Fatal(err)
 		}

cmd/dummy-object-layer_test.go

@@ -59,19 +59,19 @@ func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix,
 	return
 }
 
-func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
+func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	return
 }
 
-func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error) {
+func (api *DummyObjectLayer) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 	return
 }
 
@@ -83,15 +83,15 @@ func (api *DummyObjectLayer) ListMultipartUploads(ctx context.Context, bucket, p
 	return
 }
 
-func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (api *DummyObjectLayer) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error) {
+func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
+func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
 	return
 }
 

cmd/encryption-v1.go

@@ -43,6 +43,8 @@ var (
 	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
 	// Additional Minio errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
+	// error returned when invalid encryption parameters are specified
+	errInvalidEncryptionParameters = errors.New("The encryption parameters are not applicable to this object")
 )
 
 const (
@@ -147,6 +149,7 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 		}
 		copy(extKey[:], newKey)
 		sealedKey = objectKey.Seal(extKey, sealedKey.IV, crypto.SSEC.String(), bucket, object)
+		crypto.SSEC.CreateMetadata(metadata, sealedKey)
 		return nil
 	}
 }
@@ -713,28 +716,25 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
 // decryption succeeded.
 //
 // DecryptObjectInfo also returns whether the object is encrypted or not.
-func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
+func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, err error) {
 	// Directories are never encrypted.
 	if info.IsDir {
-		return ErrNone, false
+		return false, nil
 	}
 	// disallow X-Amz-Server-Side-Encryption header on HEAD and GET
 	if crypto.S3.IsRequested(headers) {
-		apiErr = ErrInvalidEncryptionParameters
+		err = errInvalidEncryptionParameters
 		return
 	}
-	if apiErr, encrypted = ErrNone, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSEC.IsRequested(headers) {
-		apiErr = ErrInvalidEncryptionParameters
+	if err, encrypted = nil, crypto.IsEncrypted(info.UserDefined); !encrypted && crypto.SSEC.IsRequested(headers) {
+		err = errInvalidEncryptionParameters
 	} else if encrypted {
 		if (crypto.SSEC.IsEncrypted(info.UserDefined) && !crypto.SSEC.IsRequested(headers)) ||
 			(crypto.S3.IsEncrypted(info.UserDefined) && crypto.SSEC.IsRequested(headers)) {
-			apiErr = ErrSSEEncryptedObject
+			err = errEncryptedObject
 			return
 		}
-		var err error
-		if info.Size, err = info.DecryptedSize(); err != nil {
-			apiErr = toAPIErrorCode(err)
-		}
+		info.Size, err = info.DecryptedSize()
 	}
 	return
 }

cmd/encryption-v1_test.go

@@ -521,43 +521,43 @@ func TestDecryptRequest(t *testing.T) {
 var decryptObjectInfoTests = []struct {
 	info    ObjectInfo
 	headers http.Header
-	expErr  APIErrorCode
+	expErr  error
 }{
 	{
 		info:    ObjectInfo{Size: 100},
 		headers: http.Header{},
-		expErr:  ErrNone,
+		expErr:  nil,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrNone,
+		expErr:  nil,
 	},
 	{
 		info:    ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrNone,
+		expErr:  nil,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSECSealedKey: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
 		headers: http.Header{},
-		expErr:  ErrSSEEncryptedObject,
+		expErr:  errEncryptedObject,
 	},
 	{
 		info:    ObjectInfo{Size: 100, UserDefined: map[string]string{}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrInvalidEncryptionParameters,
+		expErr:  errInvalidEncryptionParameters,
 	},
 	{
 		info:    ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256}},
 		headers: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}},
-		expErr:  ErrObjectTampered,
+		expErr:  errObjectTampered,
 	},
 }
 
 func TestDecryptObjectInfo(t *testing.T) {
 	for i, test := range decryptObjectInfoTests {
-		if err, encrypted := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
+		if encrypted, err := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
 			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
 		} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
 			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)

cmd/erasure-decode.go

@@ -127,8 +127,8 @@ func (p *parallelReader) Read() ([][]byte, error) {
 	return nil, errXLReadQuorum
 }
 
-// ReadFile reads from readers, reconstructs data if needed and writes the data to the writer.
-func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, readers []*bitrotReader, offset, length, totalLength int64) error {
+// Decode reads from readers, reconstructs data if needed and writes the data to the writer.
+func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []*bitrotReader, offset, length, totalLength int64) error {
 	if offset < 0 || length < 0 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
@@ -141,27 +141,27 @@ func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, readers
 		return nil
 	}
 
-	reader := newParallelReader(readers, s.dataBlocks, offset, totalLength, s.blockSize)
+	reader := newParallelReader(readers, e.dataBlocks, offset, totalLength, e.blockSize)
 
-	startBlock := offset / s.blockSize
-	endBlock := (offset + length) / s.blockSize
+	startBlock := offset / e.blockSize
+	endBlock := (offset + length) / e.blockSize
 
 	var bytesWritten int64
 	for block := startBlock; block <= endBlock; block++ {
 		var blockOffset, blockLength int64
 		switch {
 		case startBlock == endBlock:
-			blockOffset = offset % s.blockSize
+			blockOffset = offset % e.blockSize
 			blockLength = length
 		case block == startBlock:
-			blockOffset = offset % s.blockSize
-			blockLength = s.blockSize - blockOffset
+			blockOffset = offset % e.blockSize
+			blockLength = e.blockSize - blockOffset
 		case block == endBlock:
 			blockOffset = 0
-			blockLength = (offset + length) % s.blockSize
+			blockLength = (offset + length) % e.blockSize
 		default:
 			blockOffset = 0
-			blockLength = s.blockSize
+			blockLength = e.blockSize
 		}
 		if blockLength == 0 {
 			break
@@ -170,11 +170,11 @@ func (s ErasureStorage) ReadFile(ctx context.Context, writer io.Writer, readers
 		if err != nil {
 			return err
 		}
-		if err = s.ErasureDecodeDataBlocks(bufs); err != nil {
+		if err = e.DecodeDataBlocks(bufs); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
-		n, err := writeDataBlocks(ctx, writer, bufs, s.dataBlocks, blockOffset, blockLength)
+		n, err := writeDataBlocks(ctx, writer, bufs, e.dataBlocks, blockOffset, blockLength)
 		if err != nil {
 			return err
 		}

cmd/erasure-decode_test.go

@@ -32,7 +32,7 @@ func (d badDisk) ReadFile(volume string, path string, offset int64, buf []byte,
 	return 0, errFaultyDisk
 }
 
-var erasureReadFileTests = []struct {
+var erasureDecodeTests = []struct {
 	dataBlocks                   int
 	onDisks, offDisks            int
 	blocksize, data              int64
@@ -81,13 +81,13 @@ var erasureReadFileTests = []struct {
 	{dataBlocks: 8, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV1), data: int64(2 * blockSizeV1), offset: 11, length: int64(blockSizeV1) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false},            // 37
 }
 
-func TestErasureReadFile(t *testing.T) {
-	for i, test := range erasureReadFileTests {
+func TestErasureDecode(t *testing.T) {
+	for i, test := range erasureDecodeTests {
 		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -108,7 +108,7 @@ func TestErasureReadFile(t *testing.T) {
 		for i, disk := range disks {
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", writeAlgorithm)
 		}
-		n, err := storage.CreateFile(context.Background(), bytes.NewReader(data[:]), writers, buffer, storage.dataBlocks+1)
+		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[:]), writers, buffer, erasure.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create erasure test file: %v", i, err)
@@ -129,12 +129,12 @@ func TestErasureReadFile(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, storage.dataBlocks)
+			endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, erasure.dataBlocks)
 			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", writeAlgorithm, endOffset, writers[index].Sum())
 		}
 
 		writer := bytes.NewBuffer(nil)
-		err = storage.ReadFile(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
+		err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -157,7 +157,7 @@ func TestErasureReadFile(t *testing.T) {
 				if disk == OfflineDisk {
 					continue
 				}
-				endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, storage.dataBlocks)
+				endOffset := getErasureShardFileEndOffset(test.offset, test.length, test.data, test.blocksize, erasure.dataBlocks)
 				bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", writeAlgorithm, endOffset, writers[index].Sum())
 			}
 			for j := range disks[:test.offDisks] {
@@ -167,7 +167,7 @@ func TestErasureReadFile(t *testing.T) {
 				bitrotReaders[0] = nil
 			}
 			writer.Reset()
-			err = storage.ReadFile(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
+			err = erasure.Decode(context.Background(), writer, bitrotReaders, test.offset, test.length, test.data)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -184,10 +184,10 @@ func TestErasureReadFile(t *testing.T) {
 	}
 }
 
-// Test erasureReadFile with random offset and lengths.
+// Test erasureDecode with random offset and lengths.
 // This test is t.Skip()ed as it a long time to run, hence should be run
 // explicitly after commenting out t.Skip()
-func TestErasureReadFileRandomOffsetLength(t *testing.T) {
+func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 	// Comment the following line to run this test.
 	t.SkipNow()
 	// Initialize environment needed for the test.
@@ -201,7 +201,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 	}
 	defer setup.Remove()
 	disks := setup.disks
-	storage, err := NewErasureStorage(context.Background(), dataBlocks, parityBlocks, blockSize)
+	erasure, err := NewErasure(context.Background(), dataBlocks, parityBlocks, blockSize)
 	if err != nil {
 		t.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -226,7 +226,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 	// Create a test file to read from.
 	buffer := make([]byte, blockSize, 2*blockSize)
-	n, err := storage.CreateFile(context.Background(), bytes.NewReader(data), writers, buffer, storage.dataBlocks+1)
+	n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -239,7 +239,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 	buf := &bytes.Buffer{}
 
-	// Verify erasureReadFile() for random offsets and lengths.
+	// Verify erasure.Decode() for random offsets and lengths.
 	for i := 0; i < iterations; i++ {
 		offset := r.Int63n(length)
 		readLen := r.Int63n(length - offset)
@@ -252,10 +252,10 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(offset, readLen, length, blockSize, storage.dataBlocks)
+			endOffset := getErasureShardFileEndOffset(offset, readLen, length, blockSize, erasure.dataBlocks)
 			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", DefaultBitrotAlgorithm, endOffset, writers[index].Sum())
 		}
-		err = storage.ReadFile(context.Background(), buf, bitrotReaders, offset, readLen, length)
+		err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length)
 		if err != nil {
 			t.Fatal(err, offset, readLen)
 		}
@@ -269,14 +269,14 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
 
 // Benchmarks
 
-func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
+func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
 	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
 	disks := setup.disks
-	storage, err := NewErasureStorage(context.Background(), data, parity, blockSizeV1)
+	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -291,7 +291,7 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 
 	content := make([]byte, size)
 	buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
-	_, err = storage.CreateFile(context.Background(), bytes.NewReader(content), writers, buffer, storage.dataBlocks+1)
+	_, err = erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
 	if err != nil {
 		b.Fatalf("failed to create erasure test file: %v", err)
 	}
@@ -312,62 +312,62 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
 			if writers[index] == nil {
 				continue
 			}
-			endOffset := getErasureShardFileEndOffset(0, size, size, storage.blockSize, storage.dataBlocks)
+			endOffset := getErasureShardFileEndOffset(0, size, size, erasure.blockSize, erasure.dataBlocks)
 			bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", DefaultBitrotAlgorithm, endOffset, writers[index].Sum())
 		}
-		if err = storage.ReadFile(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size); err != nil {
+		if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size); err != nil {
 			panic(err)
 		}
 	}
 }
 
-func BenchmarkErasureReadQuick(b *testing.B) {
+func BenchmarkErasureDecodeQuick(b *testing.B) {
 	const size = 12 * 1024 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
-	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 0, size, b) })
+	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 1, size, b) })
 }
 
-func BenchmarkErasureRead_4_64KB(b *testing.B) {
+func BenchmarkErasureDecode_4_64KB(b *testing.B) {
 	const size = 64 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
-	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
-	b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 2, size, b) })
-	b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 2, 0, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 0, size, b) })
+	b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 1, size, b) })
+	b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 2, size, b) })
+	b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 2, 0, size, b) })
 }
 
-func BenchmarkErasureRead_8_20MB(b *testing.B) {
+func BenchmarkErasureDecode_8_20MB(b *testing.B) {
 	const size = 20 * 1024 * 1024
-	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 0, size, b) })
-	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 1, size, b) })
-	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 0, size, b) })
-	b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 1, size, b) })
-	b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 4, size, b) })
-	b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 2, 2, size, b) })
-	b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 4, 0, size, b) })
+	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 0, size, b) })
+	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 1, size, b) })
+	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 1, 0, size, b) })
+	b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 1, 1, size, b) })
+	b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 4, size, b) })
+	b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 2, 2, size, b) })
+	b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 4, 0, size, b) })
 }
 
-func BenchmarkErasureRead_12_30MB(b *testing.B) {
+func BenchmarkErasureDecode_12_30MB(b *testing.B) {
 	const size = 30 * 1024 * 1024
-	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 0, size, b) })
-	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 1, size, b) })
-	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 0, size, b) })
-	b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 1, size, b) })
-	b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 6, size, b) })
-	b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 3, 3, size, b) })
-	b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 6, 0, size, b) })
+	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 0, size, b) })
+	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 1, size, b) })
+	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 1, 0, size, b) })
+	b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 1, 1, size, b) })
+	b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 6, size, b) })
+	b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 3, 3, size, b) })
+	b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 6, 0, size, b) })
 }
 
-func BenchmarkErasureRead_16_40MB(b *testing.B) {
+func BenchmarkErasureDecode_16_40MB(b *testing.B) {
 	const size = 40 * 1024 * 1024
-	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 0, size, b) })
-	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 1, size, b) })
-	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 0, size, b) })
-	b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 1, size, b) })
-	b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 8, size, b) })
-	b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 4, 4, size, b) })
-	b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 8, 0, size, b) })
+	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 0, size, b) })
+	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 1, size, b) })
+	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 1, 0, size, b) })
+	b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 1, 1, size, b) })
+	b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 8, size, b) })
+	b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 4, 4, size, b) })
+	b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 8, 0, size, b) })
 }

cmd/erasure-encode.go

@@ -69,8 +69,8 @@ func (p *parallelWriter) Append(ctx context.Context, blocks [][]byte) error {
 	return reduceWriteQuorumErrs(ctx, p.errs, objectOpIgnoredErrs, p.writeQuorum)
 }
 
-// CreateFile reads from the reader, erasure-encodes the data and writes to the writers.
-func (s *ErasureStorage) CreateFile(ctx context.Context, src io.Reader, writers []*bitrotWriter, buf []byte, quorum int) (total int64, err error) {
+// Encode reads from the reader, erasure-encodes the data and writes to the writers.
+func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []*bitrotWriter, buf []byte, quorum int) (total int64, err error) {
 	writer := &parallelWriter{
 		writers:     writers,
 		writeQuorum: quorum,
@@ -90,7 +90,7 @@ func (s *ErasureStorage) CreateFile(ctx context.Context, src io.Reader, writers
 			break
 		}
 		// We take care of the situation where if n == 0 and total == 0 by creating empty data and parity files.
-		blocks, err = s.ErasureEncode(ctx, buf[:n])
+		blocks, err = e.EncodeData(ctx, buf[:n])
 		if err != nil {
 			logger.LogIf(ctx, err)
 			return 0, err

cmd/erasure-encode_test.go

@@ -34,7 +34,7 @@ func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
 
 const oneMiByte = 1 * humanize.MiByte
 
-var erasureCreateFileTests = []struct {
+var erasureEncodeTests = []struct {
 	dataBlocks                   int
 	onDisks, offDisks            int
 	blocksize, data              int64
@@ -64,14 +64,14 @@ var erasureCreateFileTests = []struct {
 	{dataBlocks: 10, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV1), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true},                // 19
 }
 
-func TestErasureCreateFile(t *testing.T) {
-	for i, test := range erasureCreateFileTests {
+func TestErasureEncode(t *testing.T) {
+	for i, test := range erasureEncodeTests {
 		setup, err := newErasureTestSetup(test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			t.Fatalf("Test %d: failed to create test setup: %v", i, err)
 		}
 		disks := setup.disks
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -90,7 +90,7 @@ func TestErasureCreateFile(t *testing.T) {
 			}
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", test.algorithm)
 		}
-		n, err := storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, storage.dataBlocks+1)
+		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but failed with: %v", i, err)
 		}
@@ -119,7 +119,7 @@ func TestErasureCreateFile(t *testing.T) {
 			if test.offDisks > 0 {
 				writers[0] = nil
 			}
-			n, err = storage.CreateFile(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, storage.dataBlocks+1)
+			n, err = erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
 			if err != nil && !test.shouldFailQuorum {
 				t.Errorf("Test %d: should pass but failed with: %v", i, err)
 			}
@@ -138,13 +138,13 @@ func TestErasureCreateFile(t *testing.T) {
 
 // Benchmarks
 
-func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
+func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
 	setup, err := newErasureTestSetup(data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create test setup: %v", err)
 	}
 	defer setup.Remove()
-	storage, err := NewErasureStorage(context.Background(), data, parity, blockSizeV1)
+	erasure, err := NewErasure(context.Background(), data, parity, blockSizeV1)
 	if err != nil {
 		b.Fatalf("failed to create ErasureStorage: %v", err)
 	}
@@ -170,50 +170,50 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
 			}
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", DefaultBitrotAlgorithm)
 		}
-		_, err := storage.CreateFile(context.Background(), bytes.NewReader(content), writers, buffer, storage.dataBlocks+1)
+		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
 		if err != nil {
 			panic(err)
 		}
 	}
 }
 
-func BenchmarkErasureWriteQuick(b *testing.B) {
+func BenchmarkErasureEncodeQuick(b *testing.B) {
 	const size = 12 * 1024 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 1, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_4_64KB(b *testing.B) {
+func BenchmarkErasureEncode_4_64KB(b *testing.B) {
 	const size = 64 * 1024
-	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
-	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
-	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
+	b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 0, size, b) })
+	b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 1, size, b) })
+	b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 1, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_8_20MB(b *testing.B) {
+func BenchmarkErasureEncode_8_20MB(b *testing.B) {
 	const size = 20 * 1024 * 1024
-	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 0, size, b) })
-	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 1, size, b) })
-	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 1, 0, size, b) })
-	b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 3, size, b) })
-	b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 3, 0, size, b) })
+	b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 0, size, b) })
+	b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 1, size, b) })
+	b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 1, 0, size, b) })
+	b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 3, size, b) })
+	b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 3, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_12_30MB(b *testing.B) {
+func BenchmarkErasureEncode_12_30MB(b *testing.B) {
 	const size = 30 * 1024 * 1024
-	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 0, size, b) })
-	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 1, size, b) })
-	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 1, 0, size, b) })
-	b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 5, size, b) })
-	b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 5, 0, size, b) })
+	b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 0, size, b) })
+	b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 1, size, b) })
+	b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 1, 0, size, b) })
+	b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 5, size, b) })
+	b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 5, 0, size, b) })
 }
 
-func BenchmarkErasureWrite_16_40MB(b *testing.B) {
+func BenchmarkErasureEncode_16_40MB(b *testing.B) {
 	const size = 40 * 1024 * 1024
-	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 0, size, b) })
-	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 1, size, b) })
-	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 1, 0, size, b) })
-	b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 7, size, b) })
-	b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 7, 0, size, b) })
+	b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 0, size, b) })
+	b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 1, size, b) })
+	b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 1, 0, size, b) })
+	b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 7, size, b) })
+	b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 7, 0, size, b) })
 }

cmd/erasure-heal.go

@@ -23,20 +23,20 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-// HealFile heals the shard files on non-nil writers. Note that the quorum passed is 1
+// Heal heals the shard files on non-nil writers. Note that the quorum passed is 1
 // as healing should continue even if it has been successful healing only one shard file.
-func (s ErasureStorage) HealFile(ctx context.Context, readers []*bitrotReader, writers []*bitrotWriter, size int64) error {
+func (e Erasure) Heal(ctx context.Context, readers []*bitrotReader, writers []*bitrotWriter, size int64) error {
 	r, w := io.Pipe()
 	go func() {
-		if err := s.ReadFile(ctx, w, readers, 0, size, size); err != nil {
+		if err := e.Decode(ctx, w, readers, 0, size, size); err != nil {
 			w.CloseWithError(err)
 			return
 		}
 		w.Close()
 	}()
-	buf := make([]byte, s.blockSize)
+	buf := make([]byte, e.blockSize)
 	// quorum is 1 because CreateFile should continue writing as long as we are writing to even 1 disk.
-	n, err := s.CreateFile(ctx, r, writers, buf, 1)
+	n, err := e.Encode(ctx, r, writers, buf, 1)
 	if err != nil {
 		return err
 	}

cmd/erasure-heal_test.go

@@ -24,7 +24,7 @@ import (
 	"testing"
 )
 
-var erasureHealFileTests = []struct {
+var erasureHealTests = []struct {
 	dataBlocks, disks int
 
 	// number of offline disks is also number of staleDisks for
@@ -60,8 +60,8 @@ var erasureHealFileTests = []struct {
 	{dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV1), size: oneMiByte * 64, algorithm: SHA256, shouldFail: false},              // 19
 }
 
-func TestErasureHealFile(t *testing.T) {
-	for i, test := range erasureHealFileTests {
+func TestErasureHeal(t *testing.T) {
+	for i, test := range erasureHealTests {
 		if test.offDisks < test.badStaleDisks {
 			// test case sanity check
 			t.Fatalf("Test %d: Bad test case - number of stale disks cannot be less than number of badstale disks", i)
@@ -73,7 +73,7 @@ func TestErasureHealFile(t *testing.T) {
 			t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
 		}
 		disks := setup.disks
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
@@ -88,7 +88,7 @@ func TestErasureHealFile(t *testing.T) {
 		for i, disk := range disks {
 			writers[i] = newBitrotWriter(disk, "testbucket", "testobject", test.algorithm)
 		}
-		_, err = storage.CreateFile(context.Background(), bytes.NewReader(data), writers, buffer, storage.dataBlocks+1)
+		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		if err != nil {
 			setup.Remove()
 			t.Fatalf("Test %d: failed to create random test data: %v", i, err)
@@ -96,7 +96,7 @@ func TestErasureHealFile(t *testing.T) {
 
 		readers := make([]*bitrotReader, len(disks))
 		for i, disk := range disks {
-			shardFilesize := getErasureShardFileSize(test.blocksize, test.size, storage.dataBlocks)
+			shardFilesize := getErasureShardFileSize(test.blocksize, test.size, erasure.dataBlocks)
 			readers[i] = newBitrotReader(disk, "testbucket", "testobject", test.algorithm, shardFilesize, writers[i].Sum())
 		}
 
@@ -126,7 +126,7 @@ func TestErasureHealFile(t *testing.T) {
 		}
 
 		// test case setup is complete - now call Healfile()
-		err = storage.HealFile(context.Background(), readers, staleWriters, test.size)
+		err = erasure.Heal(context.Background(), readers, staleWriters, test.size)
 		if err != nil && !test.shouldFail {
 			t.Errorf("Test %d: should pass but it failed with: %v", i, err)
 		}

cmd/erasure.go

@@ -23,26 +23,23 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
-// OfflineDisk represents an unavailable disk.
-var OfflineDisk StorageAPI // zero value is nil
-
-// ErasureStorage - erasure encoding details.
-type ErasureStorage struct {
-	erasure                  reedsolomon.Encoder
+// Erasure - erasure encoding details.
+type Erasure struct {
+	encoder                  reedsolomon.Encoder
 	dataBlocks, parityBlocks int
 	blockSize                int64
 }
 
-// NewErasureStorage creates a new ErasureStorage.
-func NewErasureStorage(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
+// NewErasure creates a new ErasureStorage.
+func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (e Erasure, err error) {
 	shardsize := int(ceilFrac(blockSize, int64(dataBlocks)))
 	erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize))
 	if err != nil {
 		logger.LogIf(ctx, err)
-		return s, err
+		return e, err
 	}
-	s = ErasureStorage{
-		erasure:      erasure,
+	e = Erasure{
+		encoder:      erasure,
 		dataBlocks:   dataBlocks,
 		parityBlocks: parityBlocks,
 		blockSize:    blockSize,
@@ -50,30 +47,30 @@ func NewErasureStorage(ctx context.Context, dataBlocks, parityBlocks int, blockS
 	return
 }
 
-// ErasureEncode encodes the given data and returns the erasure-coded data.
+// EncodeData encodes the given data and returns the erasure-coded data.
 // It returns an error if the erasure coding failed.
-func (s *ErasureStorage) ErasureEncode(ctx context.Context, data []byte) ([][]byte, error) {
+func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error) {
 	if len(data) == 0 {
-		return make([][]byte, s.dataBlocks+s.parityBlocks), nil
+		return make([][]byte, e.dataBlocks+e.parityBlocks), nil
 	}
-	encoded, err := s.erasure.Split(data)
+	encoded, err := e.encoder.Split(data)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
-	if err = s.erasure.Encode(encoded); err != nil {
+	if err = e.encoder.Encode(encoded); err != nil {
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
 	return encoded, nil
 }
 
-// ErasureDecodeDataBlocks decodes the given erasure-coded data.
+// DecodeDataBlocks decodes the given erasure-coded data.
 // It only decodes the data blocks but does not verify them.
 // It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
+func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
 	needsReconstruction := false
-	for _, b := range data[:s.dataBlocks] {
+	for _, b := range data[:e.dataBlocks] {
 		if b == nil {
 			needsReconstruction = true
 			break
@@ -82,16 +79,16 @@ func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
 	if !needsReconstruction {
 		return nil
 	}
-	if err := s.erasure.ReconstructData(data); err != nil {
+	if err := e.encoder.ReconstructData(data); err != nil {
 		return err
 	}
 	return nil
 }
 
-// ErasureDecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
+// DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
 // It returns an error if the decoding failed.
-func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
-	if err := s.erasure.Reconstruct(data); err != nil {
+func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
+	if err := e.encoder.Reconstruct(data); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}

cmd/erasure_test.go

@@ -25,7 +25,7 @@ import (
 	"testing"
 )
 
-var erasureDecodeTests = []struct {
+var erasureEncodeDecodeTests = []struct {
 	dataBlocks, parityBlocks   int
 	missingData, missingParity int
 	reconstructParity          bool
@@ -43,20 +43,20 @@ var erasureDecodeTests = []struct {
 	{dataBlocks: 8, parityBlocks: 4, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: false},
 }
 
-func TestErasureDecode(t *testing.T) {
+func TestErasureEncodeDecode(t *testing.T) {
 	data := make([]byte, 256)
 	if _, err := io.ReadFull(rand.Reader, data); err != nil {
 		t.Fatalf("Failed to read random data: %v", err)
 	}
-	for i, test := range erasureDecodeTests {
+	for i, test := range erasureEncodeDecodeTests {
 		buffer := make([]byte, len(data), 2*len(data))
 		copy(buffer, data)
 
-		storage, err := NewErasureStorage(context.Background(), test.dataBlocks, test.parityBlocks, blockSizeV1)
+		erasure, err := NewErasure(context.Background(), test.dataBlocks, test.parityBlocks, blockSizeV1)
 		if err != nil {
-			t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
+			t.Fatalf("Test %d: failed to create erasure: %v", i, err)
 		}
-		encoded, err := storage.ErasureEncode(context.Background(), buffer)
+		encoded, err := erasure.EncodeData(context.Background(), buffer)
 		if err != nil {
 			t.Fatalf("Test %d: failed to encode data: %v", i, err)
 		}
@@ -69,9 +69,9 @@ func TestErasureDecode(t *testing.T) {
 		}
 
 		if test.reconstructParity {
-			err = storage.ErasureDecodeDataAndParityBlocks(context.Background(), encoded)
+			err = erasure.DecodeDataAndParityBlocks(context.Background(), encoded)
 		} else {
-			err = storage.ErasureDecodeDataBlocks(encoded)
+			err = erasure.DecodeDataBlocks(encoded)
 		}
 
 		if err == nil && test.shouldFail {

cmd/fs-v1-metadata_test.go

@@ -54,7 +54,7 @@ func TestReadFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 
@@ -89,7 +89,7 @@ func TestWriteFSMetadata(t *testing.T) {
 	if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
-	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
+	if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
 

cmd/fs-v1-multipart.go

@@ -209,7 +209,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k
 // subsequent request each UUID is unique.
 //
 // Implements S3 compatible initiate multipart API.
-func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string) (string, error) {
+func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) (string, error) {
 	if err := checkNewMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return "", toObjectErr(err, bucket)
 	}
@@ -249,7 +249,7 @@ func (fs *FSObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 // object. Internally incoming data is written to '.minio.sys/tmp' location
 // and safely renamed to '.minio.sys/multipart' for reach parts.
 func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int,
-	startOffset int64, length int64, srcInfo ObjectInfo) (pi PartInfo, e error) {
+	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
 
 	if err := checkNewMultipartArgs(ctx, srcBucket, srcObject, fs); err != nil {
 		return pi, toObjectErr(err)
@@ -257,7 +257,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 
 	// Initialize pipe.
 	go func() {
-		if gerr := fs.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag); gerr != nil {
+		if gerr := fs.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil {
 			if gerr = srcInfo.Writer.Close(); gerr != nil {
 				logger.LogIf(ctx, gerr)
 				return
@@ -271,7 +271,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 		}
 	}()
 
-	partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader)
+	partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader, dstOpts)
 	if err != nil {
 		return pi, toObjectErr(err, dstBucket, dstObject)
 	}
@@ -283,7 +283,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 // an ongoing multipart transaction. Internally incoming data is
 // written to '.minio.sys/tmp' location and safely renamed to
 // '.minio.sys/multipart' for reach parts.
-func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) {
+func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, e error) {
 	if err := checkPutObjectPartArgs(ctx, bucket, object, fs); err != nil {
 		return pi, toObjectErr(err, bucket)
 	}

cmd/fs-v1-multipart_test.go

@@ -42,7 +42,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
@@ -81,7 +81,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
 
 	// Test with disk removed.
 	os.RemoveAll(disk)
-	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}); err != nil {
+	if _, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error ", err)
 		}
@@ -105,7 +105,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -114,7 +114,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
 	sha256sum := ""
 
 	fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum))
+	_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
 	if !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -136,7 +136,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
@@ -168,14 +168,14 @@ func TestCompleteMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
+	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
@@ -202,14 +202,14 @@ func TestAbortMultipartUpload(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	uploadID, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 
 	md5Hex := getMD5Hash(data)
 
-	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
+	if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
 	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
@@ -234,7 +234,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
 		t.Fatal("Cannot create bucket, err: ", err)
 	}
 
-	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
+	_, err := fs.NewMultipartUpload(context.Background(), bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}, ObjectOptions{})
 	if err != nil {
 		t.Fatal("Unexpected error ", err)
 	}

cmd/fs-v1.go

@@ -261,7 +261,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
 	storageInfo := StorageInfo{
 		Used: used,
 	}
-	storageInfo.Backend.Type = FS
+	storageInfo.Backend.Type = BackendFS
 	return storageInfo
 }
 
@@ -414,7 +414,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
 // CopyObject - copy object source object to destination object.
 // if source object and destination object are same we only
 // update metadata.
-func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo) (oi ObjectInfo, e error) {
+func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 	// Hold write lock on destination since in both cases
 	// - if source and destination are same
@@ -456,7 +456,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 
 		// Save objects' metadata in `fs.json`.
 		fsMeta := newFSMetaV1()
-		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil {
+		if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil && err != io.EOF {
 			return oi, toObjectErr(err, srcBucket, srcObject)
 		}
 
@@ -504,7 +504,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) {
+func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
 		return err
 	}
@@ -660,7 +660,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 		// Read from fs metadata only if it exists.
 		_, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile)
 		fs.rwPool.Close(fsMetaPath)
-		if rerr != nil {
+		if rerr != nil && rerr != io.EOF {
 			return oi, rerr
 		}
 	}
@@ -710,7 +710,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
 }
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
-func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, e error) {
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
 		objectLock := fs.nsMutex.NewNSLock(bucket, object)
@@ -754,7 +754,7 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
 // until EOF, writes data directly to configured filesystem path.
 // Additionally writes `fs.json` which carries the necessary metadata
 // for future object operations.
-func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
+func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
 	if err := checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
 		return ObjectInfo{}, err
 	}

cmd/fs-v1_test.go

@@ -40,7 +40,7 @@ func TestFSParentDirIsObject(t *testing.T) {
 	}
 	objectContent := "12345"
 	objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
-		mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
+		mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -124,7 +124,7 @@ func TestFSShutdown(t *testing.T) {
 
 		objectContent := "12345"
 		obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-		obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
+		obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
 		return fs, disk
 	}
 
@@ -203,7 +203,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a regular object.
-	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -212,7 +212,7 @@ func TestFSPutObject(t *testing.T) {
 	}
 
 	// With a directory object.
-	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, bucket doesn't exist")
 	}
@@ -220,11 +220,11 @@ func TestFSPutObject(t *testing.T) {
 		t.Fatalf("Expected error type BucketNotFound, got %#v", err)
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backend corruption occurred")
 	}
@@ -239,7 +239,7 @@ func TestFSPutObject(t *testing.T) {
 		}
 	}
 
-	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
 	if err == nil {
 		t.Fatal("Unexpected should fail here, backned corruption occurred")
 	}
@@ -267,7 +267,7 @@ func TestFSDeleteObject(t *testing.T) {
 	objectName := "object"
 
 	obj.MakeBucketWithLocation(context.Background(), bucketName, "")
-	obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
+	obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
 
 	// Test with invalid bucket name
 	if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) {

cmd/gateway-main.go

@@ -19,10 +19,10 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"net"
 	"net/url"
 	"os"
 	"os/signal"
-	"runtime"
 	"strings"
 	"syscall"
 
@@ -83,17 +83,6 @@ func ValidateGatewayArguments(serverAddr, endpointAddr string) error {
 		return err
 	}
 
-	if runtime.GOOS == "darwin" {
-		_, port := mustSplitHostPort(serverAddr)
-		// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
-		// to IPv6 address i.e minio will start listening on IPv6 address whereas another
-		// (non-)minio process is listening on IPv4 of given port.
-		// To avoid this error situation we check for port availability only for macOS.
-		if err := checkPortAvailability(port); err != nil {
-			return err
-		}
-	}
-
 	if endpointAddr != "" {
 		// Reject the endpoint if it points to the gateway handler itself.
 		sameTarget, err := sameLocalAddrs(endpointAddr, serverAddr)
@@ -148,6 +137,18 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Handle common env vars.
 	handleCommonEnvVars()
 
+	// Get port to listen on from gateway address
+	_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
+	if pErr != nil {
+		logger.FatalIf(pErr, "Unable to start gateway")
+	}
+
+	// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
+	// to IPv6 address ie minio will start listening on IPv6 address whereas another
+	// (non-)minio process is listening on IPv4 of given port.
+	// To avoid this error situation we check for port availability.
+	logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
+
 	// Validate if we have access, secret set through environment.
 	if !globalIsEnvCreds {
 		logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
@@ -170,6 +171,14 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalServerConfig = srvCfg
 	globalServerConfigMu.Unlock()
 
+	var cacheConfig = globalServerConfig.GetCacheConfig()
+	if len(cacheConfig.Drives) > 0 {
+		var err error
+		// initialize the new disk cache objects.
+		globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
+		logger.FatalIf(err, "Unable to initialize disk caching")
+	}
+
 	// Check and load SSL certificates.
 	var err error
 	globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()

cmd/gateway-unsupported.go

@@ -35,19 +35,19 @@ func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket str
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) {
+func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return "", NotImplemented{}
 }
 
 // CopyObjectPart copy part of object to uploadID for another object
-func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo) (pi PartInfo, err error) {
+func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return pi, NotImplemented{}
 }
 
 // PutObjectPart puts a part of object in bucket
-func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) {
+func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return pi, NotImplemented{}
 }
@@ -131,7 +131,7 @@ func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix,
 
 // CopyObject copies a blob from source container to destination container.
 func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
-	srcInfo ObjectInfo) (objInfo ObjectInfo, err error) {
+	srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return objInfo, NotImplemented{}
 }

cmd/gateway/azure/gateway-azure.go

@@ -621,7 +621,7 @@ func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	// startOffset cannot be negative.
 	if startOffset < 0 {
 		return azureToObjectError(minio.InvalidRange{}, bucket, object)
@@ -653,7 +653,7 @@ func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, sta
 
 // GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo,
 // uses zure equivalent GetBlobProperties.
-func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	err = blob.GetProperties(nil)
 	if err != nil {
@@ -674,7 +674,7 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string)
 
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
-func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
 	blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
 	if err != nil {
@@ -684,12 +684,12 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, dat
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	return a.GetObjectInfo(ctx, bucket, object)
+	return a.GetObjectInfo(ctx, bucket, object, opts)
 }
 
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Azure equivalent CopyBlob API.
-func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
 	destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
 	azureMeta, props, err := s3MetaToAzureProperties(ctx, srcInfo.UserDefined)
@@ -716,7 +716,7 @@ func (a *azureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, des
 	if err != nil {
 		return objInfo, azureToObjectError(err, srcBucket, srcObject)
 	}
-	return a.GetObjectInfo(ctx, destBucket, destObject)
+	return a.GetObjectInfo(ctx, destBucket, destObject, dstOpts)
 }
 
 // DeleteObject - Deletes a blob on azure container, uses Azure
@@ -763,7 +763,7 @@ func (a *azureObjects) checkUploadIDExists(ctx context.Context, bucketName, obje
 }
 
 // NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
-func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts minio.ObjectOptions) (uploadID string, err error) {
 	uploadID, err = getAzureUploadID()
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -787,7 +787,7 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
 }
 
 // PutObjectPart - Use Azure equivalent PutBlockWithLength.
-func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info minio.PartInfo, err error) {
+func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
 	if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
 		return info, err
 	}
@@ -1035,7 +1035,7 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 			return objInfo, azureToObjectError(err, bucket, object)
 		}
 	}
-	return a.GetObjectInfo(ctx, bucket, object)
+	return a.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy - Azure supports three types of container policies:

cmd/gateway/b2/gateway-b2.go

@@ -400,7 +400,7 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return err
@@ -417,7 +417,7 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string,
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
-func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) {
+func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return objInfo, err
@@ -508,7 +508,7 @@ func (nb *Reader) Read(p []byte) (int, error) {
 }
 
 // PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
-func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return objInfo, err
@@ -608,7 +608,7 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 // Each large file must consist of at least 2 parts, and all of the parts except the
 // last one must be at least 5MB in size. The last part must contain at least one byte.
 // For more information - https://www.backblaze.com/b2/docs/large_files.html
-func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (string, error) {
+func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (string, error) {
 	var uploadID string
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
@@ -627,7 +627,7 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 }
 
 // PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
-func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi minio.PartInfo, err error) {
+func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
 	bkt, err := l.Bucket(ctx, bucket)
 	if err != nil {
 		return pi, err
@@ -723,7 +723,7 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, b2ToObjectError(err, bucket, object, uploadID)
 	}
 
-	return l.GetObjectInfo(ctx, bucket, object)
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy - B2 supports 2 types of bucket policies:

cmd/gateway/gcs/gateway-gcs.go

@@ -20,11 +20,15 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math"
 	"net/http"
+	"path"
+	"strconv"
+
 	"os"
 	"regexp"
 	"strings"
@@ -738,7 +742,7 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
 	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@@ -827,7 +831,7 @@ func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAt
 }
 
 // GetObjectInfo - reads object info and replies back ObjectInfo
-func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) {
+func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
 	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@@ -845,7 +849,7 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st
 }
 
 // PutObject - Create a new object with the incoming data,
-func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string) (minio.ObjectInfo, error) {
+func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
 	// otherwise gcs will just return object not exist in case of non-existing bucket
 	if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
@@ -884,7 +888,7 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
 
 // CopyObject - Copies a blob from source container to destination container.
 func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
-	srcInfo minio.ObjectInfo) (minio.ObjectInfo, error) {
+	srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (minio.ObjectInfo, error) {
 
 	src := l.client.Bucket(srcBucket).Object(srcObject)
 	dst := l.client.Bucket(destBucket).Object(destObject)
@@ -913,7 +917,7 @@ func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object str
 }
 
 // NewMultipartUpload - upload object in multiple parts
-func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string) (uploadID string, err error) {
+func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
 	// generate new uploadid
 	uploadID = minio.MustGetUUID()
 
@@ -936,14 +940,84 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
 	return uploadID, nil
 }
 
-// ListMultipartUploads - lists all multipart uploads.
+// ListMultipartUploads - lists the (first) multipart upload for an object
+// matched _exactly_ by the prefix
 func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (minio.ListMultipartsInfo, error) {
+	// List objects under <bucket>/gcsMinioMultipartPathV1
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
+		Prefix: gcsMinioMultipartPathV1,
+	})
+
+	var uploads []minio.MultipartInfo
+
+	for {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			break
+		}
+
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return minio.ListMultipartsInfo{
+				KeyMarker:      keyMarker,
+				UploadIDMarker: uploadIDMarker,
+				MaxUploads:     maxUploads,
+				Prefix:         prefix,
+				Delimiter:      delimiter,
+			}, gcsToObjectError(err)
+		}
+
+		// Skip entries other than gcs.json
+		if !strings.HasSuffix(attrs.Name, gcsMinioMultipartMeta) {
+			continue
+		}
+
+		// Extract multipart upload information from gcs.json
+		obj := l.client.Bucket(bucket).Object(attrs.Name)
+		objReader, rErr := obj.NewReader(ctx)
+		if rErr != nil {
+			logger.LogIf(ctx, rErr)
+			return minio.ListMultipartsInfo{}, rErr
+		}
+		defer objReader.Close()
+
+		var mpMeta gcsMultipartMetaV1
+		dec := json.NewDecoder(objReader)
+		decErr := dec.Decode(&mpMeta)
+		if decErr != nil {
+			logger.LogIf(ctx, decErr)
+			return minio.ListMultipartsInfo{}, decErr
+		}
+
+		if prefix == mpMeta.Object {
+			// Extract uploadId
+			// E.g minio.sys.tmp/multipart/v1/d063ad89-fdc4-4ea3-a99e-22dba98151f5/gcs.json
+			components := strings.SplitN(attrs.Name, "/", 5)
+			if len(components) != 5 {
+				compErr := errors.New("Invalid multipart upload format")
+				logger.LogIf(ctx, compErr)
+				return minio.ListMultipartsInfo{}, compErr
+			}
+			upload := minio.MultipartInfo{
+				Object:    mpMeta.Object,
+				UploadID:  components[3],
+				Initiated: attrs.Created,
+			}
+			uploads = []minio.MultipartInfo{upload}
+			break
+		}
+	}
+
 	return minio.ListMultipartsInfo{
-		KeyMarker:      keyMarker,
-		UploadIDMarker: uploadIDMarker,
-		MaxUploads:     maxUploads,
-		Prefix:         prefix,
-		Delimiter:      delimiter,
+		KeyMarker:          keyMarker,
+		UploadIDMarker:     uploadIDMarker,
+		MaxUploads:         maxUploads,
+		Prefix:             prefix,
+		Delimiter:          delimiter,
+		Uploads:            uploads,
+		NextKeyMarker:      "",
+		NextUploadIDMarker: "",
+		IsTruncated:        false,
 	}, nil
 }
 
@@ -956,7 +1030,7 @@ func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key
 }
 
 // PutObjectPart puts a part of object in bucket
-func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (minio.PartInfo, error) {
+func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader, opts minio.ObjectOptions) (minio.PartInfo, error) {
 	if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
 		return minio.PartInfo{}, err
 	}
@@ -987,9 +1061,91 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
 
 }
 
-// ListObjectParts returns all object parts for specified object in specified bucket
+// gcsGetPartInfo returns PartInfo of a given object part
+func gcsGetPartInfo(ctx context.Context, attrs *storage.ObjectAttrs) (minio.PartInfo, error) {
+	components := strings.SplitN(attrs.Name, "/", 5)
+	if len(components) != 5 {
+		logger.LogIf(ctx, errors.New("Invalid multipart upload format"))
+		return minio.PartInfo{}, errors.New("Invalid multipart upload format")
+	}
+
+	partComps := strings.SplitN(components[4], ".", 2)
+	if len(partComps) != 2 {
+		logger.LogIf(ctx, errors.New("Invalid multipart part format"))
+		return minio.PartInfo{}, errors.New("Invalid multipart part format")
+	}
+
+	partNum, pErr := strconv.Atoi(partComps[0])
+	if pErr != nil {
+		logger.LogIf(ctx, pErr)
+		return minio.PartInfo{}, errors.New("Invalid part number")
+	}
+
+	return minio.PartInfo{
+		PartNumber:   partNum,
+		LastModified: attrs.Updated,
+		Size:         attrs.Size,
+		ETag:         partComps[1],
+	}, nil
+}
+
+//  ListObjectParts returns all object parts for specified object in specified bucket
 func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) {
-	return minio.ListPartsInfo{}, l.checkUploadIDExists(ctx, bucket, key, uploadID)
+	it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
+		Prefix: path.Join(gcsMinioMultipartPathV1, uploadID),
+	})
+
+	var (
+		count     int
+		partInfos []minio.PartInfo
+	)
+
+	isTruncated := true
+	for count < maxParts {
+		attrs, err := it.Next()
+		if err == iterator.Done {
+			isTruncated = false
+			break
+		}
+
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return minio.ListPartsInfo{}, gcsToObjectError(err)
+		}
+
+		if strings.HasSuffix(attrs.Name, gcsMinioMultipartMeta) {
+			continue
+		}
+
+		partInfo, pErr := gcsGetPartInfo(ctx, attrs)
+		if pErr != nil {
+			logger.LogIf(ctx, pErr)
+			return minio.ListPartsInfo{}, pErr
+		}
+
+		if partInfo.PartNumber <= partNumberMarker {
+			continue
+		}
+
+		partInfos = append(partInfos, partInfo)
+		count++
+	}
+
+	nextPartNumberMarker := 0
+	if isTruncated {
+		nextPartNumberMarker = partInfos[maxParts-1].PartNumber
+	}
+
+	return minio.ListPartsInfo{
+		Bucket:               bucket,
+		Object:               key,
+		UploadID:             uploadID,
+		PartNumberMarker:     partNumberMarker,
+		NextPartNumberMarker: nextPartNumberMarker,
+		MaxParts:             maxParts,
+		Parts:                partInfos,
+		IsTruncated:          isTruncated,
+	}, nil
 }
 
 // Called by AbortMultipartUpload and CompleteMultipartUpload for cleaning up.

cmd/gateway/manta/gateway-manta.go

@@ -117,12 +117,16 @@ EXAMPLES:
 }
 
 func mantaGatewayMain(ctx *cli.Context) {
-	// Validate gateway arguments.
-	host := ctx.Args().First()
-	// Validate gateway arguments.
-	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
+	args := ctx.Args()
+	if !ctx.Args().Present() {
+		args = cli.Args{"https://us-east.manta.joyent.com"}
+	}
 
-	minio.StartGateway(ctx, &Manta{host})
+	// Validate gateway arguments.
+	logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), args.First()), "Invalid argument")
+
+	// Start the gateway..
+	minio.StartGateway(ctx, &Manta{args.First()})
 }
 
 // Manta implements Gateway.
@@ -139,17 +143,22 @@ func (g *Manta) Name() string {
 // talk to manta remote backend.
 func (g *Manta) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
 	var err error
+	var secure bool
 	var signer authentication.Signer
 	var endpoint = defaultMantaURL
 	ctx := context.Background()
 
 	if g.host != "" {
-		endpoint, _, err = minio.ParseGatewayEndpoint(g.host)
+		endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
 		if err != nil {
 			return nil, err
 		}
+		if secure {
+			endpoint = "https://" + endpoint
+		} else {
+			endpoint = "http://" + endpoint
+		}
 	}
-
 	if overrideRoot, ok := os.LookupEnv("MANTA_ROOT"); ok {
 		mantaRoot = overrideRoot
 	}
@@ -504,7 +513,7 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 // indicates the total length of the object.
 //
 // https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	// Start offset cannot be negative.
 	if startOffset < 0 {
 		logger.LogIf(ctx, fmt.Errorf("Unexpected error"))
@@ -536,7 +545,7 @@ func (t *tritonObjects) GetObject(ctx context.Context, bucket, object string, st
 // uses Triton equivalent GetBlobProperties.
 //
 // https://apidocs.joyent.com/manta/api.html#GetObject
-func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	info, err := t.client.Objects().GetInfo(ctx, &storage.GetInfoInput{
 		ObjectPath: path.Join(mantaRoot, bucket, object),
 	})
@@ -574,7 +583,7 @@ func (d dummySeeker) Seek(offset int64, whence int) (int64, error) {
 // CreateBlockBlobFromReader.
 //
 // https://apidocs.joyent.com/manta/api.html#PutObject
-func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{
 		ContentLength: uint64(data.Size()),
 		ObjectPath:    path.Join(mantaRoot, bucket, object),
@@ -593,14 +602,14 @@ func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, da
 		return objInfo, err
 	}
 
-	return t.GetObjectInfo(ctx, bucket, object)
+	return t.GetObjectInfo(ctx, bucket, object, opts)
 }
 
 // CopyObject - Copies a blob from source container to destination container.
 // Uses Manta Snaplinks API.
 //
 // https://apidocs.joyent.com/manta/api.html#PutSnapLink
-func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	if err = t.client.SnapLinks().Put(ctx, &storage.PutSnapLinkInput{
 		SourcePath: path.Join(mantaRoot, srcBucket, srcObject),
 		LinkPath:   path.Join(mantaRoot, destBucket, destObject),
@@ -609,7 +618,7 @@ func (t *tritonObjects) CopyObject(ctx context.Context, srcBucket, srcObject, de
 		return objInfo, err
 	}
 
-	return t.GetObjectInfo(ctx, destBucket, destObject)
+	return t.GetObjectInfo(ctx, destBucket, destObject, dstOpts)
 }
 
 // DeleteObject - Delete a blob in Manta, uses Triton equivalent DeleteBlob API.

cmd/gateway/oss/gateway-oss.go

@@ -21,7 +21,6 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"strconv"
 	"strings"
@@ -553,7 +552,7 @@ func ossGetObject(ctx context.Context, client *oss.Client, bucket, key string, s
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string) error {
+func (l *ossObjects) GetObject(ctx context.Context, bucket, key string, startOffset, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	return ossGetObject(ctx, l.Client, bucket, key, startOffset, length, writer, etag)
 }
 
@@ -603,7 +602,7 @@ func ossGetObjectInfo(ctx context.Context, client *oss.Client, bucket, object st
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo.
-func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	return ossGetObjectInfo(ctx, l.Client, bucket, object)
 }
 
@@ -631,12 +630,12 @@ func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string
 }
 
 // PutObject creates a new object with the incoming data.
-func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	return ossPutObject(ctx, l.Client, bucket, object, data, metadata)
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
-func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	bkt, err := l.Client.Bucket(srcBucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -660,7 +659,7 @@ func (l *ossObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		logger.LogIf(ctx, err)
 		return objInfo, ossToObjectError(err, srcBucket, srcObject)
 	}
-	return l.GetObjectInfo(ctx, dstBucket, dstObject)
+	return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts)
 }
 
 // DeleteObject deletes a blob in bucket.
@@ -726,7 +725,7 @@ func (l *ossObjects) ListMultipartUploads(ctx context.Context, bucket, prefix, k
 }
 
 // NewMultipartUpload upload object in multiple parts.
-func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error) {
+func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
 	bkt, err := l.Client.Bucket(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -749,7 +748,7 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
 }
 
 // PutObjectPart puts a part of object in bucket.
-func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, err error) {
+func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
 	bkt, err := l.Client.Bucket(bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -816,11 +815,8 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 		return lupr, err
 	}
 
-	defer func() {
-		// always drain output (response body)
-		io.CopyN(ioutil.Discard, resp.Body, 512)
-		resp.Body.Close()
-	}()
+	// always drain output (response body)
+	defer minio.CloseResponse(resp.Body)
 
 	err = xml.NewDecoder(resp.Body).Decode(&lupr)
 	if err != nil {
@@ -832,7 +828,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
 
 	bkt, err := l.Client.Bucket(destBucket)
 	if err != nil {
@@ -958,7 +954,7 @@ func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 		return oi, ossToObjectError(err, bucket, object)
 	}
 
-	return l.GetObjectInfo(ctx, bucket, object)
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy sets policy on bucket.

cmd/gateway/s3/gateway-s3.go

@@ -307,12 +307,14 @@ func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 //
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
-func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, o minio.ObjectOptions) error {
 	if length < 0 && length != -1 {
 		return minio.ErrorRespToObjectError(minio.InvalidRange{}, bucket, key)
 	}
 
 	opts := miniogo.GetObjectOptions{}
+	opts.ServerSideEncryption = o.ServerSideEncryption
+
 	if startOffset >= 0 && length >= 0 {
 		if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
 			logger.LogIf(ctx, err)
@@ -333,8 +335,8 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
-func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string) (objInfo minio.ObjectInfo, err error) {
-	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{})
+func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{miniogo.GetObjectOptions{ServerSideEncryption: opts.ServerSideEncryption}})
 	if err != nil {
 		return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -343,8 +345,8 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 }
 
 // PutObject creates a new object with the incoming data,
-func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
-	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata))
+func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata), opts.ServerSideEncryption)
 	if err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -353,7 +355,7 @@ func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string,
 }
 
 // CopyObject copies an object from source bucket to a destination bucket.
-func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo) (objInfo minio.ObjectInfo, err error) {
+func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	// Set this header such that following CopyObject() always sets the right metadata on the destination.
 	// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
 	// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
@@ -363,7 +365,7 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
 	if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
 	}
-	return l.GetObjectInfo(ctx, dstBucket, dstObject)
+	return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts)
 }
 
 // DeleteObject deletes a blob in bucket
@@ -387,9 +389,9 @@ func (l *s3Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 }
 
 // NewMultipartUpload upload object in multiple parts
-func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string) (uploadID string, err error) {
+func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
 	// Create PutObject options
-	opts := miniogo.PutObjectOptions{UserMetadata: metadata}
+	opts := miniogo.PutObjectOptions{UserMetadata: metadata, ServerSideEncryption: o.ServerSideEncryption}
 	uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
 	if err != nil {
 		return uploadID, minio.ErrorRespToObjectError(err, bucket, object)
@@ -398,8 +400,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 }
 
 // PutObjectPart puts a part of object in bucket
-func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi minio.PartInfo, e error) {
-	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString())
+func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
+	info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), opts.ServerSideEncryption)
 	if err != nil {
 		return pi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -410,7 +412,7 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str
 // CopyObjectPart creates a part in a multipart upload by copying
 // existing object or a part of it.
 func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
-	partID int, startOffset, length int64, srcInfo minio.ObjectInfo) (p minio.PartInfo, err error) {
+	partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
 
 	srcInfo.UserDefined = map[string]string{
 		"x-amz-copy-source-if-match": srcInfo.ETag,
@@ -448,7 +450,7 @@ func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		return oi, minio.ErrorRespToObjectError(err, bucket, object)
 	}
 
-	return l.GetObjectInfo(ctx, bucket, object)
+	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
 }
 
 // SetBucketPolicy sets policy on bucket

cmd/gateway/sia/gateway-sia.go

@@ -233,13 +233,13 @@ func apiGet(ctx context.Context, addr, call, apiPassword string) (*http.Response
 		return nil, err
 	}
 	if resp.StatusCode == http.StatusNotFound {
-		resp.Body.Close()
+		minio.CloseResponse(resp.Body)
 		logger.LogIf(ctx, MethodNotSupported{call})
 		return nil, MethodNotSupported{call}
 	}
 	if non2xx(resp.StatusCode) {
 		err := decodeError(resp)
-		resp.Body.Close()
+		minio.CloseResponse(resp.Body)
 		logger.LogIf(ctx, err)
 		return nil, err
 	}
@@ -266,13 +266,13 @@ func apiPost(ctx context.Context, addr, call, vals, apiPassword string) (*http.R
 	}
 
 	if resp.StatusCode == http.StatusNotFound {
-		resp.Body.Close()
+		minio.CloseResponse(resp.Body)
 		return nil, MethodNotSupported{call}
 	}
 
 	if non2xx(resp.StatusCode) {
 		err := decodeError(resp)
-		resp.Body.Close()
+		minio.CloseResponse(resp.Body)
 		return nil, err
 	}
 	return resp, nil
@@ -285,7 +285,7 @@ func post(ctx context.Context, addr, call, vals, apiPassword string) error {
 	if err != nil {
 		return err
 	}
-	resp.Body.Close()
+	minio.CloseResponse(resp.Body)
 	return nil
 }
 
@@ -295,7 +295,7 @@ func list(ctx context.Context, addr string, apiPassword string, obj *renterFiles
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
+	defer minio.CloseResponse(resp.Body)
 
 	if resp.StatusCode == http.StatusNoContent {
 		logger.LogIf(ctx, fmt.Errorf("Expecting a response, but API returned %s", resp.Status))
@@ -313,7 +313,7 @@ func get(ctx context.Context, addr, call, apiPassword string) error {
 	if err != nil {
 		return err
 	}
-	resp.Body.Close()
+	minio.CloseResponse(resp.Body)
 	return nil
 }
 
@@ -431,7 +431,7 @@ func (s *siaObjects) ListObjects(ctx context.Context, bucket string, prefix stri
 	return loi, nil
 }
 
-func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
+func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	dstFile := path.Join(s.TempDir, minio.MustGetUUID())
 	defer os.Remove(dstFile)
 
@@ -512,7 +512,7 @@ func (s *siaObjects) findSiaObject(ctx context.Context, bucket, object string) (
 }
 
 // GetObjectInfo reads object info and replies back ObjectInfo
-func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object string) (minio.ObjectInfo, error) {
+func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
 	so, err := s.findSiaObject(ctx, bucket, object)
 	if err != nil {
 		return minio.ObjectInfo{}, err
@@ -529,7 +529,7 @@ func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object st
 }
 
 // PutObject creates a new object with the incoming data,
-func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
+func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
 	srcFile := path.Join(s.TempDir, minio.MustGetUUID())
 	writer, err := os.Create(srcFile)
 	if err != nil {

cmd/handler-utils.go

@@ -189,6 +189,15 @@ func extractReqParams(r *http.Request) map[string]string {
 	}
 }
 
+// Extract response elements to be sent with event notifiation.
+func extractRespElements(w http.ResponseWriter) map[string]string {
+
+	return map[string]string{
+		"content-length": w.Header().Get("Content-Length"),
+		// Add more fields here.
+	}
+}
+
 // Trims away `aws-chunked` from the content-encoding header if present.
 // Streaming signature clients can have custom content-encoding such as
 // `aws-chunked,gzip` here we need to only save `gzip`.

cmd/http-stats.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
@@ -26,22 +27,50 @@ import (
 	"go.uber.org/atomic"
 )
 
+func getRequestResource(r *http.Request) string {
+	if r == nil {
+		// http.Request is nil when non-HTTP data (like TLS record) is read/written.
+		return ""
+	}
+
+	if globalDomainName != "" {
+		host := r.Header.Get("Host")
+		if strings.HasSuffix(host, "."+globalDomainName) {
+			return "/" + strings.TrimSuffix(host, "."+globalDomainName) + r.URL.Path
+		}
+	}
+
+	return r.URL.Path
+}
+
 // ConnStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
 type ConnStats struct {
-	totalInputBytes  atomic.Uint64
-	totalOutputBytes atomic.Uint64
+	totalInputBytes     atomic.Uint64
+	totalOutputBytes    atomic.Uint64
+	totalRPCInputBytes  atomic.Uint64
+	totalRPCOutputBytes atomic.Uint64
 }
 
 // Increase total input bytes
-func (s *ConnStats) incInputBytes(n int) {
-	s.totalInputBytes.Add(uint64(n))
+func (s *ConnStats) incInputBytes(r *http.Request, n int) {
+	resource := getRequestResource(r)
+	if resource == minioReservedBucketPath || strings.HasPrefix(resource, minioReservedBucketPath+"/") {
+		s.totalRPCInputBytes.Add(uint64(n))
+	} else {
+		s.totalInputBytes.Add(uint64(n))
+	}
 }
 
 // Increase total output bytes
-func (s *ConnStats) incOutputBytes(n int) {
-	s.totalOutputBytes.Add(uint64(n))
+func (s *ConnStats) incOutputBytes(r *http.Request, n int) {
+	resource := getRequestResource(r)
+	if resource == minioReservedBucketPath || strings.HasPrefix(resource, minioReservedBucketPath+"/") {
+		s.totalRPCOutputBytes.Add(uint64(n))
+	} else {
+		s.totalOutputBytes.Add(uint64(n))
+	}
 }
 
 // Return total input bytes

cmd/http/bufconn.go

@@ -19,17 +19,28 @@ package http
 import (
 	"bufio"
 	"net"
+	"net/http"
 	"time"
 )
 
 // BufConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout.
 type BufConn struct {
 	QuirkConn
-	bufReader              *bufio.Reader // buffered reader wraps reader in net.Conn.
-	readTimeout            time.Duration // sets the read timeout in the connection.
-	writeTimeout           time.Duration // sets the write timeout in the connection.
-	updateBytesReadFunc    func(int)     // function to be called to update bytes read.
-	updateBytesWrittenFunc func(int)     // function to be called to update bytes written.
+	bufReader              *bufio.Reader            // buffered reader wraps reader in net.Conn.
+	readTimeout            time.Duration            // sets the read timeout in the connection.
+	writeTimeout           time.Duration            // sets the write timeout in the connection.
+	request                *http.Request            // HTTP request information.
+	updateBytesReadFunc    func(*http.Request, int) // function to be called to update bytes read.
+	updateBytesWrittenFunc func(*http.Request, int) // function to be called to update bytes written.
+}
+
+func (c *BufConn) setRequest(request *http.Request) {
+	c.request = request
+}
+
+func (c *BufConn) setUpdateFuncs(updateBytesReadFunc, updateBytesWrittenFunc func(*http.Request, int)) {
+	c.updateBytesReadFunc = updateBytesReadFunc
+	c.updateBytesWrittenFunc = updateBytesWrittenFunc
 }
 
 // Sets read timeout
@@ -70,7 +81,7 @@ func (c *BufConn) Read(b []byte) (n int, err error) {
 	c.setReadTimeout()
 	n, err = c.bufReader.Read(b)
 	if err == nil && c.updateBytesReadFunc != nil {
-		c.updateBytesReadFunc(n)
+		c.updateBytesReadFunc(c.request, n)
 	}
 
 	return n, err
@@ -81,21 +92,18 @@ func (c *BufConn) Write(b []byte) (n int, err error) {
 	c.setWriteTimeout()
 	n, err = c.Conn.Write(b)
 	if err == nil && c.updateBytesWrittenFunc != nil {
-		c.updateBytesWrittenFunc(n)
+		c.updateBytesWrittenFunc(c.request, n)
 	}
 
 	return n, err
 }
 
 // newBufConn - creates a new connection object wrapping net.Conn.
-func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration,
-	updateBytesReadFunc, updateBytesWrittenFunc func(int)) *BufConn {
+func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration) *BufConn {
 	return &BufConn{
-		QuirkConn:              QuirkConn{Conn: c},
-		bufReader:              bufio.NewReader(c),
-		readTimeout:            readTimeout,
-		writeTimeout:           writeTimeout,
-		updateBytesReadFunc:    updateBytesReadFunc,
-		updateBytesWrittenFunc: updateBytesWrittenFunc,
+		QuirkConn:    QuirkConn{Conn: c},
+		bufReader:    bufio.NewReader(c),
+		readTimeout:  readTimeout,
+		writeTimeout: writeTimeout,
 	}
 }

cmd/http/bufconn_test.go

@@ -48,7 +48,7 @@ func TestBuffConnReadTimeout(t *testing.T) {
 		if terr != nil {
 			t.Fatalf("failed to accept new connection. %v", terr)
 		}
-		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second, nil, nil)
+		bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second)
 		defer bufconn.Close()
 
 		// Read a line

cmd/http/listen_nix.go

@@ -18,13 +18,20 @@
 
 package http
 
-import "github.com/valyala/tcplisten"
+import (
+	"net"
+
+	"github.com/valyala/tcplisten"
+)
 
 var cfg = &tcplisten.Config{
-	ReusePort:   true,
 	DeferAccept: true,
 	FastOpen:    true,
+	// Bump up the soMaxConn value from 128 to 2048 to
+	// handle large incoming concurrent requests.
+	Backlog: 2048,
 }
 
 // Unix listener with special TCP options.
 var listen = cfg.NewListener
+var fallbackListen = net.Listen

cmd/http/listen_others.go

@@ -22,3 +22,4 @@ import "net"
 
 // Windows, plan9 specific listener.
 var listen = net.Listen
+var fallbackListen = net.Listen

cmd/http/listener.go

@@ -23,6 +23,7 @@ import (
 	"io"
 	"net"
 	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"sync"
@@ -72,6 +73,44 @@ func isHTTPMethod(s string) bool {
 	return false
 }
 
+func getResourceHost(bufConn *BufConn, maxHeaderBytes, methodLen int) (resource string, host string, err error) {
+	var data []byte
+	for dataLen := 0; methodLen+dataLen < maxHeaderBytes; dataLen += 8 {
+		if data, err = bufConn.Peek(methodLen + dataLen); err != nil {
+			return "", "", err
+		}
+
+		tokens := strings.Split(string(data), "\n")
+		if len(tokens) < 2 {
+			continue
+		}
+
+		if resource == "" {
+			resource = strings.SplitN(tokens[0][methodLen:], " ", 2)[0]
+		}
+
+		for _, token := range tokens[1:] {
+			if token == "" || !strings.HasSuffix(token, "\r") {
+				continue
+			}
+
+			// HTTP headers are case insensitive, so we should simply convert
+			// each tokens to their lower case form to match 'host' header.
+			token = strings.ToLower(token)
+			if strings.HasPrefix(token, "host: ") {
+				host = strings.TrimPrefix(strings.TrimSuffix(token, "\r"), "host: ")
+				return resource, host, nil
+			}
+		}
+
+		if tokens[len(tokens)-1] == "\r" {
+			break
+		}
+	}
+
+	return resource, host, nil
+}
+
 type acceptResult struct {
 	conn net.Conn
 	err  error
@@ -87,8 +126,9 @@ type httpListener struct {
 	tcpKeepAliveTimeout    time.Duration
 	readTimeout            time.Duration
 	writeTimeout           time.Duration
-	updateBytesReadFunc    func(int) // function to be called to update bytes read in BufConn.
-	updateBytesWrittenFunc func(int) // function to be called to update bytes written in BufConn.
+	maxHeaderBytes         int
+	updateBytesReadFunc    func(*http.Request, int) // function to be called to update bytes read in BufConn.
+	updateBytesWrittenFunc func(*http.Request, int) // function to be called to update bytes written in BufConn.
 }
 
 // isRoutineNetErr returns true if error is due to a network timeout,
@@ -134,8 +174,7 @@ func (listener *httpListener) start() {
 		tcpConn.SetKeepAlive(true)
 		tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout)
 
-		bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout,
-			listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
+		bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout)
 
 		// Peek bytes of maximum length of all HTTP methods.
 		data, err := bufconn.Peek(methodMaxLen)
@@ -158,14 +197,39 @@ func (listener *httpListener) start() {
 		// Return bufconn if read data is a valid HTTP method.
 		tokens := strings.SplitN(string(data), " ", 2)
 		if isHTTPMethod(tokens[0]) {
-			if listener.tlsConfig == nil {
-				send(acceptResult{bufconn, nil}, doneCh)
-			} else {
+			if listener.tlsConfig != nil {
 				// As TLS is configured and we got plain text HTTP request,
 				// return 403 (forbidden) error.
 				bufconn.Write(sslRequiredErrMsg)
 				bufconn.Close()
+				return
 			}
+
+			var resource, host string
+			if resource, host, err = getResourceHost(bufconn, listener.maxHeaderBytes, len(tokens[0])+1); err != nil {
+				if !isRoutineNetErr(err) {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
+					reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
+					ctx := logger.SetReqInfo(context.Background(), reqInfo)
+					logger.LogIf(ctx, err)
+				}
+				bufconn.Close()
+				return
+			}
+
+			header := make(http.Header)
+			if host != "" {
+				header.Add("Host", host)
+			}
+			bufconn.setRequest(&http.Request{
+				Method: tokens[0],
+				URL:    &url.URL{Path: resource},
+				Host:   bufconn.LocalAddr().String(),
+				Header: header,
+			})
+			bufconn.setUpdateFuncs(listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
+
+			send(acceptResult{bufconn, nil}, doneCh)
 			return
 		}
 
@@ -182,8 +246,7 @@ func (listener *httpListener) start() {
 			}
 
 			// Check whether the connection contains HTTP request or not.
-			bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout,
-				listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
+			bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout)
 
 			// Peek bytes of maximum length of all HTTP methods.
 			data, err = bufconn.Peek(methodMaxLen)
@@ -201,6 +264,30 @@ func (listener *httpListener) start() {
 			// Return bufconn if read data is a valid HTTP method.
 			tokens := strings.SplitN(string(data), " ", 2)
 			if isHTTPMethod(tokens[0]) {
+				var resource, host string
+				if resource, host, err = getResourceHost(bufconn, listener.maxHeaderBytes, len(tokens[0])+1); err != nil {
+					if !isRoutineNetErr(err) {
+						reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
+						reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
+						ctx := logger.SetReqInfo(context.Background(), reqInfo)
+						logger.LogIf(ctx, err)
+					}
+					bufconn.Close()
+					return
+				}
+
+				header := make(http.Header)
+				if host != "" {
+					header.Add("Host", host)
+				}
+				bufconn.setRequest(&http.Request{
+					Method: tokens[0],
+					URL:    &url.URL{Path: resource},
+					Host:   bufconn.LocalAddr().String(),
+					Header: header,
+				})
+				bufconn.setUpdateFuncs(listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
+
 				send(acceptResult{bufconn, nil}, doneCh)
 				return
 			}
@@ -296,8 +383,9 @@ func newHTTPListener(serverAddrs []string,
 	tcpKeepAliveTimeout time.Duration,
 	readTimeout time.Duration,
 	writeTimeout time.Duration,
-	updateBytesReadFunc func(int),
-	updateBytesWrittenFunc func(int)) (listener *httpListener, err error) {
+	maxHeaderBytes int,
+	updateBytesReadFunc func(*http.Request, int),
+	updateBytesWrittenFunc func(*http.Request, int)) (listener *httpListener, err error) {
 
 	var tcpListeners []*net.TCPListener
 
@@ -316,7 +404,9 @@ func newHTTPListener(serverAddrs []string,
 	for _, serverAddr := range serverAddrs {
 		var l net.Listener
 		if l, err = listen("tcp4", serverAddr); err != nil {
-			return nil, err
+			if l, err = fallbackListen("tcp4", serverAddr); err != nil {
+				return nil, err
+			}
 		}
 
 		tcpListener, ok := l.(*net.TCPListener)
@@ -333,6 +423,7 @@ func newHTTPListener(serverAddrs []string,
 		tcpKeepAliveTimeout:    tcpKeepAliveTimeout,
 		readTimeout:            readTimeout,
 		writeTimeout:           writeTimeout,
+		maxHeaderBytes:         maxHeaderBytes,
 		updateBytesReadFunc:    updateBytesReadFunc,
 		updateBytesWrittenFunc: updateBytesWrittenFunc,
 	}

cmd/http/listener_test.go

@@ -19,13 +19,11 @@ package http
 import (
 	"bufio"
 	"bytes"
-	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"io"
 	"net"
-	"runtime"
+	"net/http"
 	"strconv"
 	"strings"
 	"sync"
@@ -195,27 +193,6 @@ func TestIsHTTPMethod(t *testing.T) {
 }
 
 func TestNewHTTPListener(t *testing.T) {
-	errMsg := ": no such host"
-
-	remoteAddrErrMsgIP := "cannot bind to \"93.184.216.34:65432\": cannot assign requested address"
-	if runtime.GOOS == "windows" {
-		remoteAddrErrMsgIP = "listen tcp 93.184.216.34:65432: bind: The requested address is not valid in its context."
-	}
-	remoteAddrErrMsgHost := "cannot bind to \"example.org:65432\": cannot assign requested address"
-	if runtime.GOOS == "windows" {
-		remoteAddrErrMsgHost = "listen tcp 93.184.216.34:65432: bind: The requested address is not valid in its context."
-	}
-
-	remoteMissingErr := "address unknown-host: missing port in address"
-	if runtime.GOOS == "windows" {
-		remoteMissingErr = "listen tcp: address unknown-host: missing port in address"
-	}
-
-	remoteUnknownErr := "lookup unknown-host" + errMsg
-	if runtime.GOOS == "wpindows" {
-		remoteUnknownErr = "listen tcp: lookup unknown-host" + errMsg
-	}
-
 	tlsConfig := getTLSConfig(t)
 
 	testCases := []struct {
@@ -224,19 +201,18 @@ func TestNewHTTPListener(t *testing.T) {
 		tcpKeepAliveTimeout    time.Duration
 		readTimeout            time.Duration
 		writeTimeout           time.Duration
-		updateBytesReadFunc    func(int)
-		updateBytesWrittenFunc func(int)
-		errorLogFunc           func(context.Context, error)
-		expectedErr            error
+		updateBytesReadFunc    func(*http.Request, int)
+		updateBytesWrittenFunc func(*http.Request, int)
+		expectedErr            bool
 	}{
-		{[]string{"93.184.216.34:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, errors.New(remoteAddrErrMsgIP)},
-		{[]string{"example.org:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, errors.New(remoteAddrErrMsgHost)},
-		{[]string{"unknown-host"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, errors.New(remoteMissingErr)},
-		{[]string{"unknown-host:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, errors.New(remoteUnknownErr)},
-		{[]string{"localhost:65432", "93.184.216.34:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, errors.New(remoteAddrErrMsgIP)},
-		{[]string{"localhost:65432", "unknown-host:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, errors.New(remoteUnknownErr)},
-		{[]string{"localhost:0"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, nil},
-		{[]string{"localhost:0"}, tlsConfig, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, nil, nil},
+		{[]string{"93.184.216.34:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
+		{[]string{"example.org:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
+		{[]string{"unknown-host"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
+		{[]string{"unknown-host:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
+		{[]string{"localhost:65432", "93.184.216.34:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
+		{[]string{"localhost:65432", "unknown-host:65432"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, true},
+		{[]string{"localhost:0"}, nil, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, false},
+		{[]string{"localhost:0"}, tlsConfig, time.Duration(0), time.Duration(0), time.Duration(0), nil, nil, false},
 	}
 
 	for _, testCase := range testCases {
@@ -246,26 +222,17 @@ func TestNewHTTPListener(t *testing.T) {
 			testCase.tcpKeepAliveTimeout,
 			testCase.readTimeout,
 			testCase.writeTimeout,
+			DefaultMaxHeaderBytes,
 			testCase.updateBytesReadFunc,
 			testCase.updateBytesWrittenFunc,
 		)
 
-		if testCase.expectedErr == nil {
+		if !testCase.expectedErr {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
 			}
 		} else if err == nil {
 			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else {
-			var match bool
-			if strings.HasSuffix(testCase.expectedErr.Error(), errMsg) {
-				match = strings.HasSuffix(err.Error(), errMsg)
-			} else {
-				match = (testCase.expectedErr.Error() == err.Error())
-			}
-			if !match {
-				t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-			}
 		}
 
 		if err == nil {
@@ -297,6 +264,7 @@ func TestHTTPListenerStartClose(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -344,6 +312,7 @@ func TestHTTPListenerAddr(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -388,6 +357,7 @@ func TestHTTPListenerAddrs(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -414,17 +384,20 @@ func TestHTTPListenerAccept(t *testing.T) {
 	nonLoopBackIP := getNonLoopBackIP(t)
 
 	testCases := []struct {
-		serverAddrs []string
-		tlsConfig   *tls.Config
-		request     string
-		reply       string
+		serverAddrs         []string
+		tlsConfig           *tls.Config
+		request             string
+		reply               string
+		expectedRequestLine string
 	}{
-		{[]string{"localhost:0"}, nil, "GET / HTTP/1.0\n", "200 OK\n"},
-		{[]string{nonLoopBackIP + ":0"}, nil, "POST / HTTP/1.0\n", "200 OK\n"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, "CONNECT \n", "200 OK\n"},
-		{[]string{"localhost:0"}, tlsConfig, "GET / HTTP/1.0\n", "200 OK\n"},
-		{[]string{nonLoopBackIP + ":0"}, tlsConfig, "POST / HTTP/1.0\n", "200 OK\n"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, "CONNECT \n", "200 OK\n"},
+		{[]string{"localhost:0"}, nil, "GET / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "GET / HTTP/1.0\r\n"},
+		{[]string{nonLoopBackIP + ":0"}, nil, "POST / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "POST / HTTP/1.0\r\n"},
+		{[]string{nonLoopBackIP + ":0"}, nil, "HEAD / HTTP/1.0\r\nhost: example.org\r\n\r\n", "200 OK\r\n", "HEAD / HTTP/1.0\r\n"},
+		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, "CONNECT \r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT \r\n"},
+		{[]string{"localhost:0"}, tlsConfig, "GET / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "GET / HTTP/1.0\r\n"},
+		{[]string{nonLoopBackIP + ":0"}, tlsConfig, "POST / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "POST / HTTP/1.0\r\n"},
+		{[]string{nonLoopBackIP + ":0"}, tlsConfig, "HEAD / HTTP/1.0\r\nhost: example.org\r\n\r\n", "200 OK\r\n", "HEAD / HTTP/1.0\r\n"},
+		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, "CONNECT \r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT \r\n"},
 	}
 
 	for i, testCase := range testCases {
@@ -434,6 +407,7 @@ func TestHTTPListenerAccept(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -463,13 +437,13 @@ func TestHTTPListenerAccept(t *testing.T) {
 				t.Fatalf("Test %d: accept: expected = <nil>, got = %v", i+1, err)
 			}
 
-			request, err := bufio.NewReader(serverConn).ReadString('\n')
+			requestLine, err := bufio.NewReader(serverConn).ReadString('\n')
 			if err != nil {
 				t.Fatalf("Test %d: request read: expected = <nil>, got = %v", i+1, err)
 			}
 
-			if testCase.request != request {
-				t.Fatalf("Test %d: request: expected = %v, got = %v", i+1, testCase.request, request)
+			if requestLine != testCase.expectedRequestLine {
+				t.Fatalf("Test %d: request: expected = %v, got = %v", i+1, testCase.expectedRequestLine, requestLine)
 			}
 
 			if _, err = io.WriteString(serverConn, testCase.reply); err != nil {
@@ -513,6 +487,7 @@ func TestHTTPListenerAcceptPeekError(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -566,6 +541,7 @@ func TestHTTPListenerAcceptTLSError(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -632,6 +608,7 @@ func TestHTTPListenerAcceptError(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -757,6 +734,7 @@ func TestHTTPListenerAcceptParallel(t *testing.T) {
 			time.Duration(0),
 			time.Duration(0),
 			time.Duration(0),
+			DefaultMaxHeaderBytes,
 			nil,
 			nil,
 		)
@@ -765,8 +743,8 @@ func TestHTTPListenerAcceptParallel(t *testing.T) {
 		}
 
 		for _, serverAddr := range listener.Addrs() {
-			go connect(i, serverAddr.String(), testCase.tlsConfig != nil, true, "GET /1 HTTP/1.0\n", testCase.reply)
-			go connect(i, serverAddr.String(), testCase.tlsConfig != nil, false, "GET /2 HTTP/1.0\n", testCase.reply)
+			go connect(i, serverAddr.String(), testCase.tlsConfig != nil, true, "GET /1 HTTP/1.0\r\nHost: example.org\r\nr\n", testCase.reply)
+			go connect(i, serverAddr.String(), testCase.tlsConfig != nil, false, "GET /2 HTTP/1.0\r\nHost: example.org\r\n\r\n", testCase.reply)
 
 			var wg sync.WaitGroup
 
@@ -775,14 +753,14 @@ func TestHTTPListenerAcceptParallel(t *testing.T) {
 				t.Fatalf("Test %d: accept: expected = <nil>, got = %v", i+1, err)
 			}
 			wg.Add(1)
-			go handleConnection(i, &wg, serverConn, "GET /2 HTTP/1.0\n", testCase.reply)
+			go handleConnection(i, &wg, serverConn, "GET /2 HTTP/1.0\r\n", testCase.reply)
 
 			serverConn, err = listener.Accept()
 			if err != nil {
 				t.Fatalf("Test %d: accept: expected = <nil>, got = %v", i+1, err)
 			}
 			wg.Add(1)
-			go handleConnection(i, &wg, serverConn, "GET /1 HTTP/1.0\n", testCase.reply)
+			go handleConnection(i, &wg, serverConn, "GET /1 HTTP/1.0\r\n", testCase.reply)
 
 			wg.Wait()
 		}

cmd/http/server.go

@@ -51,15 +51,15 @@ const (
 // Server - extended http.Server supports multiple addresses to serve and enhanced connection handling.
 type Server struct {
 	http.Server
-	Addrs                  []string      // addresses on which the server listens for new connection.
-	ShutdownTimeout        time.Duration // timeout used for graceful server shutdown.
-	TCPKeepAliveTimeout    time.Duration // timeout used for underneath TCP connection.
-	UpdateBytesReadFunc    func(int)     // function to be called to update bytes read in bufConn.
-	UpdateBytesWrittenFunc func(int)     // function to be called to update bytes written in bufConn.
-	listenerMutex          *sync.Mutex   // to guard 'listener' field.
-	listener               *httpListener // HTTP listener for all 'Addrs' field.
-	inShutdown             uint32        // indicates whether the server is in shutdown or not
-	requestCount           int32         // counter holds no. of request in progress.
+	Addrs                  []string                 // addresses on which the server listens for new connection.
+	ShutdownTimeout        time.Duration            // timeout used for graceful server shutdown.
+	TCPKeepAliveTimeout    time.Duration            // timeout used for underneath TCP connection.
+	UpdateBytesReadFunc    func(*http.Request, int) // function to be called to update bytes read in bufConn.
+	UpdateBytesWrittenFunc func(*http.Request, int) // function to be called to update bytes written in bufConn.
+	listenerMutex          *sync.Mutex              // to guard 'listener' field.
+	listener               *httpListener            // HTTP listener for all 'Addrs' field.
+	inShutdown             uint32                   // indicates whether the server is in shutdown or not
+	requestCount           int32                    // counter holds no. of request in progress.
 }
 
 // GetRequestCount - returns number of request in progress.
@@ -91,6 +91,7 @@ func (srv *Server) Start() (err error) {
 		tcpKeepAliveTimeout,
 		readTimeout,
 		writeTimeout,
+		srv.MaxHeaderBytes,
 		updateBytesReadFunc,
 		updateBytesWrittenFunc,
 	)

cmd/logger/http.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
+	"io"
+	"io/ioutil"
 	"net/http"
 )
 
@@ -54,6 +56,8 @@ func (h *HTTPTarget) startHTTPLogger() {
 				continue
 			}
 			if resp.Body != nil {
+				buf := make([]byte, 512)
+				io.CopyBuffer(ioutil.Discard, resp.Body, buf)
 				resp.Body.Close()
 			}
 		}

cmd/logger/reqinfo.go

@@ -68,6 +68,29 @@ func (r *ReqInfo) AppendTags(key string, val string) *ReqInfo {
 	return r
 }
 
+// SetTags - sets key/val to ReqInfo.tags
+func (r *ReqInfo) SetTags(key string, val string) *ReqInfo {
+	if r == nil {
+		return nil
+	}
+	r.Lock()
+	defer r.Unlock()
+	// Search of tag key already exists in tags
+	var updated bool
+	for _, tag := range r.tags {
+		if tag.Key == key {
+			tag.Val = val
+			updated = true
+			break
+		}
+	}
+	if !updated {
+		// Append to the end of tags list
+		r.tags = append(r.tags, KeyVal{key, val})
+	}
+	return r
+}
+
 // GetTags - returns the user defined tags
 func (r *ReqInfo) GetTags() []KeyVal {
 	if r == nil {

cmd/metrics.go

@@ -124,7 +124,7 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
 
 	var totalDisks, offlineDisks int
 	// Setting totalDisks to 1 and offlineDisks to 0 in FS mode
-	if s.Backend.Type == FS {
+	if s.Backend.Type == BackendFS {
 		totalDisks = 1
 		offlineDisks = 0
 	} else {

cmd/naughty-disk_test.go

@@ -52,6 +52,10 @@ func (d *naughtyDisk) IsOnline() bool {
 	return d.disk.IsOnline()
 }
 
+func (d *naughtyDisk) LastError() (err error) {
+	return nil
+}
+
 func (d *naughtyDisk) Close() (err error) {
 	if err = d.calcError(); err != nil {
 		return err

cmd/notification.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"net/url"
 	"path"
+	"strings"
 	"sync"
 	"time"
 
@@ -46,7 +47,13 @@ func (sys *NotificationSys) GetARNList() []string {
 	arns := []string{}
 	region := globalServerConfig.GetRegion()
 	for _, targetID := range sys.targetList.List() {
-		arns = append(arns, targetID.ToARN(region).String())
+		// httpclient target is part of ListenBucketNotification
+		// which doesn't need to be listed as part of the ARN list
+		// This list is only meant for external targets, filter
+		// this out pro-actively.
+		if !strings.HasPrefix(targetID.ID, "httpclient+") {
+			arns = append(arns, targetID.ToARN(region).String())
+		}
 	}
 
 	return arns
@@ -192,11 +199,17 @@ func (sys *NotificationSys) AddRemoteTarget(bucketName string, target event.Targ
 	if targetMap == nil {
 		targetMap = make(map[event.TargetID]event.RulesMap)
 	}
-	targetMap[target.ID()] = rulesMap.Clone()
+
+	rulesMap = rulesMap.Clone()
+	targetMap[target.ID()] = rulesMap
 	sys.bucketRemoteTargetRulesMap[bucketName] = targetMap
+
+	rulesMap = rulesMap.Clone()
+	rulesMap.Add(sys.bucketRulesMap[bucketName])
+	sys.bucketRulesMap[bucketName] = rulesMap
+
 	sys.Unlock()
 
-	sys.AddRulesMap(bucketName, rulesMap)
 	return nil
 }
 
@@ -341,8 +354,12 @@ func (sys *NotificationSys) AddRulesMap(bucketName string, rulesMap event.RulesM
 		rulesMap.Add(targetRulesMap)
 	}
 
-	rulesMap.Add(sys.bucketRulesMap[bucketName])
-	sys.bucketRulesMap[bucketName] = rulesMap
+	// Do not add for an empty rulesMap.
+	if len(rulesMap) == 0 {
+		delete(sys.bucketRulesMap, bucketName)
+	} else {
+		sys.bucketRulesMap[bucketName] = rulesMap
+	}
 }
 
 // RemoveRulesMap - removes rules map for bucket name.
@@ -439,13 +456,14 @@ func NewNotificationSys(config *serverConfig, endpoints EndpointList) *Notificat
 }
 
 type eventArgs struct {
-	EventName  event.Name
-	BucketName string
-	Object     ObjectInfo
-	ReqParams  map[string]string
-	Host       string
-	Port       string
-	UserAgent  string
+	EventName    event.Name
+	BucketName   string
+	Object       ObjectInfo
+	ReqParams    map[string]string
+	RespElements map[string]string
+	Host         string
+	Port         string
+	UserAgent    string
 }
 
 // ToEvent - converts to notification event.
@@ -464,6 +482,13 @@ func (args eventArgs) ToEvent() event.Event {
 	eventTime := UTCNow()
 	uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
 
+	respElements := map[string]string{
+		"x-amz-request-id":        uniqueID,
+		"x-minio-origin-endpoint": getOriginEndpoint(), // Minio specific custom elements.
+	}
+	if args.RespElements["content-length"] != "" {
+		respElements["content-length"] = args.RespElements["content-length"]
+	}
 	newEvent := event.Event{
 		EventVersion:      "2.0",
 		EventSource:       "minio:s3",
@@ -472,10 +497,7 @@ func (args eventArgs) ToEvent() event.Event {
 		EventName:         args.EventName,
 		UserIdentity:      event.Identity{creds.AccessKey},
 		RequestParameters: args.ReqParams,
-		ResponseElements: map[string]string{
-			"x-amz-request-id":        uniqueID,
-			"x-minio-origin-endpoint": getOriginEndpoint(), // Minio specific custom elements.
-		},
+		ResponseElements:  respElements,
 		S3: event.Metadata{
 			SchemaVersion:   "1.0",
 			ConfigurationID: "Config",

cmd/object-api-datatypes.go

@@ -31,9 +31,9 @@ type BackendType int
 const (
 	Unknown BackendType = iota
 	// Filesystem backend.
-	FS
-	// Multi disk Erasure (single, distributed) backend.
-	Erasure
+	BackendFS
+	// Multi disk BackendErasure (single, distributed) backend.
+	BackendErasure
 	// Add your own backend.
 )
 

cmd/object-api-deleteobject_test.go

@@ -0,0 +1,120 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"testing"
+)
+
+// Wrapper for calling DeleteObject tests for both XL multiple disks and single node setup.
+func TestDeleteObject(t *testing.T) {
+	ExecObjectLayerTest(t, testDeleteObject)
+}
+
+// Unit test for DeleteObject in general.
+func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
+
+	type objectUpload struct {
+		name    string
+		content string
+	}
+
+	testCases := []struct {
+		bucketName         string
+		objectToUploads    []objectUpload
+		pathToDelete       string
+		objectsAfterDelete []string
+	}{
+		// Test 1: removes an object and checks it is the only object
+		// that has been deleted.
+		{
+			"bucket1",
+			[]objectUpload{{"object0", "content"}, {"object1", "content"}},
+			"object0",
+			[]string{"object1"},
+		},
+		// Test 2: remove an object inside a directory and checks it is deleted
+		// with its parent since this former becomes empty
+		{
+			"bucket2",
+			[]objectUpload{{"object0", "content"}, {"dir/object1", "content"}},
+			"dir/object1",
+			[]string{"object0"},
+		},
+		// Test 3: remove an object inside a directory and checks if it is deleted
+		// but other sibling object in the same directory still exists
+		{
+			"bucket3",
+			[]objectUpload{{"dir/object1", "content"}, {"dir/object2", "content"}},
+			"dir/object1",
+			[]string{"dir/object2"},
+		},
+		// Test 4: remove a non empty directory and checks it has no effect
+		{
+			"bucket4",
+			[]objectUpload{{"object0", "content"}, {"dir/object1", "content"}},
+			"dir/",
+			[]string{"dir/object1", "object0"},
+		},
+		// Test 4: Remove an empty directory and checks it is really removed
+		{
+			"bucket5",
+			[]objectUpload{{"object0", "content"}, {"dir/", ""}},
+			"dir/",
+			[]string{"object0"},
+		},
+	}
+
+	for i, testCase := range testCases {
+
+		err := obj.MakeBucketWithLocation(context.Background(), testCase.bucketName, "")
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+
+		for _, object := range testCase.objectToUploads {
+			md5Bytes := md5.Sum([]byte(object.content))
+			_, err = obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetHashReader(t, bytes.NewBufferString(object.content),
+				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), nil, ObjectOptions{})
+			if err != nil {
+				t.Fatalf("%s : %s", instanceType, err.Error())
+			}
+		}
+
+		// TODO: check the error in the future
+		_ = obj.DeleteObject(context.Background(), testCase.bucketName, testCase.pathToDelete)
+
+		result, err := obj.ListObjects(context.Background(), testCase.bucketName, "", "", "", 1000)
+		if err != nil {
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+		}
+
+		if len(result.Objects) != len(testCase.objectsAfterDelete) {
+			t.Errorf("Test %d: %s: mismatch number of objects after delete, expected = %d, found = %d", i+1, instanceType, len(testCase.objectsAfterDelete), len(result.Objects))
+		}
+
+		for idx := range result.Objects {
+			if result.Objects[idx].Name != testCase.objectsAfterDelete[idx] {
+				t.Errorf("Test %d: %s: Unexpected object found after delete, found = `%v`", i+1, instanceType, result.Objects[idx].Name)
+			}
+		}
+	}
+}

cmd/object-api-getobject_test.go

@@ -39,6 +39,8 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// Setup for the tests.
 	bucketName := getRandomBucketName()
 	objectName := "test-object"
+	emptyDirName := "test-empty-dir/"
+
 	// create bucket.
 	err := obj.MakeBucketWithLocation(context.Background(), bucketName, "")
 	// Stop the test if creation of the bucket fails.
@@ -53,7 +55,10 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	bytesData := []struct {
 		byteData []byte
 	}{
+		// Regular data
 		{generateBytesData(6 * humanize.MiByte)},
+		// Empty data for empty directory
+		{},
 	}
 	// set of inputs for uploading the objects before tests for downloading is done.
 	putObjectInputs := []struct {
@@ -65,11 +70,12 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}{
 		// case - 1.
 		{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
+		{bucketName, emptyDirName, int64(len(bytesData[1].byteData)), bytesData[1].byteData, make(map[string]string)},
 	}
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -79,6 +85,7 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	buffers := []*bytes.Buffer{
 		new(bytes.Buffer),
 		new(bytes.Buffer),
+		new(bytes.Buffer),
 	}
 
 	// test cases with set of inputs
@@ -107,45 +114,48 @@ func testGetObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		// Test case - 5.
 		// Case with invalid object names.
 		{bucketName, "", 0, 0, nil, nil, false, []byte(""), fmt.Errorf("%s", "Object name invalid: "+bucketName+"#")},
-		//	Test case - 7.
+		//	Test case - 6.
 		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[0], NewEOFWriter(buffers[0], 100), false, []byte{}, io.EOF},
 		// Test case with start offset set to 0 and length set to size of the object.
 		// Fetching the entire object.
-		// 	Test case - 8.
+		// 	Test case - 7.
 		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], true, bytesData[0].byteData, nil},
 		// Test case with `length` parameter set to a negative value.
-		// Test case - 9.
+		// Test case - 8.
 		{bucketName, objectName, 0, int64(-1), buffers[1], buffers[1], true, bytesData[0].byteData, nil},
 		// Test case with content-range 1 to objectSize .
-		// Test case - 10.
+		// Test case - 9.
 		{bucketName, objectName, 1, int64(len(bytesData[0].byteData) - 1), buffers[1], buffers[1], true, bytesData[0].byteData[1:], nil},
 		// Test case with content-range 100 to objectSize - 100.
-		// Test case - 11.
+		// Test case - 10.
 		{bucketName, objectName, 100, int64(len(bytesData[0].byteData) - 200), buffers[1], buffers[1], true,
 			bytesData[0].byteData[100 : len(bytesData[0].byteData)-100], nil},
 		// Test case with offset greater than the size of the object
-		// Test case - 12.
+		// Test case - 11.
 		{bucketName, objectName, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), buffers[0],
 			NewEOFWriter(buffers[0], 100), false, []byte{},
 			InvalidRange{int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
 		// Test case with offset greater than the size of the object.
-		// Test case - 13.
+		// Test case - 12.
 		{bucketName, objectName, -1, int64(len(bytesData[0].byteData)), buffers[0], new(bytes.Buffer), false, []byte{}, errUnexpected},
 		// Test case length parameter is more than the object size.
-		// Test case - 14.
+		// Test case - 13.
 		{bucketName, objectName, 0, int64(len(bytesData[0].byteData) + 1), buffers[1], buffers[1], false, bytesData[0].byteData,
 			InvalidRange{0, int64(len(bytesData[0].byteData) + 1), int64(len(bytesData[0].byteData))}},
 		// Test case with offset + length > objectSize parameter set to a negative value.
-		// Test case - 15.
+		// Test case - 14.
 		{bucketName, objectName, 2, int64(len(bytesData[0].byteData)), buffers[1], buffers[1], false, bytesData[0].byteData,
 			InvalidRange{2, int64(len(bytesData[0].byteData)), int64(len(bytesData[0].byteData))}},
 		// Test case with the writer set to nil.
-		// Test case - 16.
+		// Test case - 15.
 		{bucketName, objectName, 0, int64(len(bytesData[0].byteData)), buffers[1], nil, false, bytesData[0].byteData, errUnexpected},
+		// Test case - 16.
+		// Test case when it is an empty directory
+		{bucketName, emptyDirName, 0, int64(len(bytesData[1].byteData)), buffers[2], buffers[2], true, bytesData[1].byteData, nil},
 	}
 
 	for i, testCase := range testCases {
-		err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "")
+		err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{})
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 		}
@@ -210,7 +220,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -255,7 +265,7 @@ func testGetObjectPermissionDenied(obj ObjectLayer, instanceType string, disks [
 			}
 		}
 
-		err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "")
+		err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{})
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 		}
@@ -323,7 +333,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
 	// iterate through the above set of inputs and upkoad the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData["etag"], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -408,7 +418,7 @@ func testGetObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []str
 	}
 
 	for i, testCase := range testCases {
-		err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "")
+		err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, testCase.startOffset, testCase.length, testCase.writer, "", ObjectOptions{})
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 		}

cmd/object-api-getobjectinfo_test.go

@@ -34,14 +34,23 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
-	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetHashReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), nil)
+	opts := ObjectOptions{}
+	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetHashReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
+
+	// Put an empty directory
+	_, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetHashReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), nil, opts)
+	if err != nil {
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
 	resultCases := []ObjectInfo{
 		// ObjectInfo -1.
 		// ObjectName set to a existing object in the test case (Test case 14).
 		{Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false},
+		{Bucket: "test-getobjectinfo", Name: "Asia/empty-dir/", ContentType: "application/octet-stream", IsDir: true},
 	}
 	testCases := []struct {
 		bucketName string
@@ -70,9 +79,10 @@ func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		{"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false},
 		// Valid case with existing object (Test number 12).
 		{"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true},
+		{"test-getobjectinfo", "Asia/empty-dir/", resultCases[1], nil, true},
 	}
 	for i, testCase := range testCases {
-		result, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName)
+		result, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName, opts)
 		if err != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
 		}

cmd/object-api-interface.go

@@ -20,11 +20,17 @@ import (
 	"context"
 	"io"
 
+	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
 
+// ObjectOptions represents object options for ObjectLayer operations
+type ObjectOptions struct {
+	ServerSideEncryption encrypt.ServerSide
+}
+
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
 	// Storage operations.
@@ -40,18 +46,19 @@ type ObjectLayer interface {
 	ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 
 	// Object operations.
-	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
-	GetObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error)
-	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error)
-	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo) (objInfo ObjectInfo, err error)
+
+	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
+	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string) error
 
 	// Multipart operations.
 	ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error)
-	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string) (uploadID string, err error)
+	NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
 	CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int,
-		startOffset int64, length int64, srcInfo ObjectInfo) (info PartInfo, err error)
-	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error)
+		startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error)
+	PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
 	ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error)
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)

cmd/object-api-listobjects_test.go

@@ -66,11 +66,12 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		{"obj0", "obj0", nil},
 		{"obj1", "obj1", nil},
 		{"obj2", "obj2", nil},
+		{"z-empty-dir/", "", nil},
 	}
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
 		_, err = obj.PutObject(context.Background(), testBuckets[0], object.name, mustGetHashReader(t, bytes.NewBufferString(object.content),
-			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), object.meta)
+			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), object.meta, ObjectOptions{})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -95,6 +96,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-1.
@@ -191,6 +193,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-10.
@@ -202,6 +205,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-11.
@@ -211,6 +215,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-12.
@@ -219,6 +224,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-13.
@@ -232,6 +238,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-14.
@@ -248,6 +255,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-15.
@@ -262,6 +270,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-16.
@@ -275,6 +284,7 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				{Name: "obj0"},
 				{Name: "obj1"},
 				{Name: "obj2"},
+				{Name: "z-empty-dir/"},
 			},
 		},
 		// ListObjectsResult-17.
@@ -452,8 +462,8 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		{"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true},
 		// Setting maxKeys to a very large value (17).
 		{"empty-bucket", "", "", "", 111100000, ListObjectsInfo{}, nil, true},
-		// Testing for all 7 objects in the bucket (18).
-		{"test-bucket-list-object", "", "", "", 9, resultCases[0], nil, true},
+		// Testing for all 10 objects in the bucket (18).
+		{"test-bucket-list-object", "", "", "", 10, resultCases[0], nil, true},
 		//Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (19).
 		{"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true},
 		// Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (20).
@@ -472,11 +482,11 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 		{"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true},
 		{"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true},
 		// Testing with marker, but without prefix and truncation (31-35).
-		{"test-bucket-list-object", "", "newPrefix0", "", 5, resultCases[9], nil, true},
-		{"test-bucket-list-object", "", "newPrefix1", "", 4, resultCases[10], nil, true},
-		{"test-bucket-list-object", "", "obj0", "", 2, resultCases[11], nil, true},
-		{"test-bucket-list-object", "", "obj1", "", 1, resultCases[12], nil, true},
-		{"test-bucket-list-object", "", "man", "", 10, resultCases[13], nil, true},
+		{"test-bucket-list-object", "", "newPrefix0", "", 6, resultCases[9], nil, true},
+		{"test-bucket-list-object", "", "newPrefix1", "", 5, resultCases[10], nil, true},
+		{"test-bucket-list-object", "", "obj0", "", 4, resultCases[11], nil, true},
+		{"test-bucket-list-object", "", "obj1", "", 2, resultCases[12], nil, true},
+		{"test-bucket-list-object", "", "man", "", 11, resultCases[13], nil, true},
 		// Marker being set to a value which is greater than and all object names when sorted (36).
 		// Expected to send an empty response in this case.
 		{"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true},
@@ -554,8 +564,9 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
 				if testCase.result.Objects[j].Name != result.Objects[j].Name {
 					t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name)
 				}
-				if result.Objects[j].ETag == "" {
-					t.Errorf("Test %d: %s: Expected ETag to be not empty, but found empty instead", i+1, instanceType)
+				//FIXME: we should always check for ETag
+				if result.Objects[j].ETag == "" && !strings.HasSuffix(result.Objects[j].Name, slashSeparator) {
+					t.Errorf("Test %d: %s: Expected ETag to be not empty, but found empty instead (%v)", i+1, instanceType, result.Objects[j].Name)
 				}
 
 			}
@@ -606,7 +617,7 @@ func BenchmarkListObjects(b *testing.B) {
 	// Insert objects to be listed and benchmarked later.
 	for i := 0; i < 20000; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), bucket, key, mustGetHashReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), nil)
+		_, err = obj.PutObject(context.Background(), bucket, key, mustGetHashReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), nil, ObjectOptions{})
 		if err != nil {
 			b.Fatal(err)
 		}

cmd/object-api-multipart_test.go

@@ -40,15 +40,15 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
 
 	bucket := "minio-bucket"
 	object := "minio-object"
-
-	_, err := obj.NewMultipartUpload(context.Background(), "--", object, nil)
+	opts := ObjectOptions{}
+	_, err := obj.NewMultipartUpload(context.Background(), "--", object, nil, opts)
 	if err == nil {
 		t.Fatalf("%s: Expected to fail since bucket name is invalid.", instanceType)
 	}
 
 	errMsg := "Bucket not found: minio-bucket"
 	// opearation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist.
-	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil)
+	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
 	if err == nil {
 		t.Fatalf("%s: Expected to fail since the NewMultipartUpload is intialized on a non-existent bucket.", instanceType)
 	}
@@ -63,12 +63,12 @@ func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestEr
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	_, err = obj.NewMultipartUpload(context.Background(), bucket, "\\", nil)
+	_, err = obj.NewMultipartUpload(context.Background(), bucket, "\\", nil, opts)
 	if err == nil {
 		t.Fatalf("%s: Expected to fail since object name is invalid.", instanceType)
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
@@ -94,7 +94,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 
 	bucket := "minio-bucket"
 	object := "minio-object"
-
+	opts := ObjectOptions{}
 	// Create bucket before intiating NewMultipartUpload.
 	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
@@ -102,7 +102,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
@@ -148,7 +148,7 @@ func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestE
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
-	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil)
+	_, err = obj.NewMultipartUpload(context.Background(), bucket, object, nil, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
@@ -184,7 +184,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	}
 
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, ObjectOptions{})
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -221,7 +221,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum))
+		_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -235,7 +235,7 @@ func testPutObjectPartDiskNotFound(obj ObjectLayer, instanceType string, disks [
 
 	// Object part upload should fail with quorum not available.
 	testCase := createPartCases[len(createPartCases)-1]
-	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum))
+	_, err = obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), ObjectOptions{})
 	if err == nil {
 		t.Fatalf("Test %s: expected to fail but passed instead", instanceType)
 	}
@@ -256,7 +256,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 	// Generating cases for which the PutObjectPart fails.
 	bucket := "minio-bucket"
 	object := "minio-object"
-
+	opts := ObjectOptions{}
 	// Create bucket before intiating NewMultipartUpload.
 	err := obj.MakeBucketWithLocation(context.Background(), bucket, "")
 	if err != nil {
@@ -264,7 +264,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -354,7 +354,7 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 
 	// Validate all the test cases.
 	for i, testCase := range testCases {
-		actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, testCase.inputSHA256))
+		actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, testCase.inputSHA256), opts)
 		// All are test cases above are expected to fail.
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
@@ -389,7 +389,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	bucketNames := []string{"minio-bucket", "minio-2-bucket", "minio-3-bucket"}
 	objectNames := []string{"minio-object-1.txt", "minio-object.txt", "neymar-1.jpeg", "neymar.jpeg", "parrot-1.png", "parrot.png"}
 	uploadIDs := []string{}
-
+	opts := ObjectOptions{}
 	// bucketnames[0].
 	// objectNames[0].
 	// uploadIds [0].
@@ -400,7 +400,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -420,7 +420,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	for i := 0; i < 3; i++ {
 		// Initiate Multipart Upload on bucketNames[1] for the same object 3 times.
 		//  Used to test the listing for the case of multiple uploadID's for a given object.
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], nil)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], nil, opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("%s : %s", instanceType, err.Error())
@@ -442,7 +442,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	//  Used to test the listing for the case of multiple objects for a given bucket.
 	for i := 0; i < 6; i++ {
 		var uploadID string
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], nil)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], nil, opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("%s : %s", instanceType, err.Error())
@@ -488,7 +488,7 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum))
+		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1273,8 +1273,9 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
+	opts := ObjectOptions{}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1308,7 +1309,7 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum))
+		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1507,7 +1508,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 	bucketNames := []string{"minio-bucket", "minio-2-bucket"}
 	objectNames := []string{"minio-object-1.txt"}
 	uploadIDs := []string{}
-
+	opts := ObjectOptions{}
 	// bucketnames[0].
 	// objectNames[0].
 	// uploadIds [0].
@@ -1518,7 +1519,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -1549,7 +1550,7 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, testCase := range createPartCases {
-		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum))
+		_, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetHashReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.intputDataSize, testCase.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -1763,8 +1764,9 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err)
 	}
+	opts := ObjectOptions{}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], map[string]string{"X-Amz-Meta-Id": "id"})
+	uploadID, err = obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], map[string]string{"X-Amz-Meta-Id": "id"}, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err)
@@ -1799,7 +1801,7 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 	sha256sum := ""
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, part := range parts {
-		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, sha256sum))
+		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, sha256sum), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err)
 		}

cmd/object-api-putobject_test.go

@@ -162,7 +162,7 @@ func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandl
 	}
 
 	for i, testCase := range testCases {
-		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta)
+		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256), testCase.inputMeta, ObjectOptions{})
 		if actualErr != nil && testCase.expectedError == nil {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i+1, instanceType, actualErr.Error())
 		}
@@ -235,7 +235,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
 
 	sha256sum := ""
 	for i, testCase := range testCases {
-		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta)
+		objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
 		if actualErr != nil && testCase.shouldPass {
 			t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", i+1, instanceType, actualErr.Error())
 		}
@@ -284,7 +284,7 @@ func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, di
 		InsufficientWriteQuorum{},
 	}
 
-	_, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta)
+	_, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetHashReader(t, bytes.NewReader(testCase.inputData), testCase.intputDataSize, testCase.inputMeta["etag"], sha256sum), testCase.inputMeta, ObjectOptions{})
 	if actualErr != nil && testCase.shouldPass {
 		t.Errorf("Test %d: %s: Expected to pass, but failed with: <ERROR> %s.", len(testCases)+1, instanceType, actualErr.Error())
 	}
@@ -316,7 +316,7 @@ func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disk
 
 	data := []byte("hello, world")
 	// Create object.
-	_, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucket, object, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, ObjectOptions{})
 	if err != nil {
 		// Failed to create object, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -347,9 +347,9 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 		// Failed to create newbucket, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
-
+	opts := ObjectOptions{}
 	// Initiate Multipart Upload on the above created bucket.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucket, object, nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -361,7 +361,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 	md5Writer.Write(fiveMBBytes)
 	etag1 := hex.EncodeToString(md5Writer.Sum(nil))
 	sha256sum := ""
-	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum))
+	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 1, mustGetHashReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum), opts)
 	if err != nil {
 		// Failed to upload object part, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
@@ -372,7 +372,7 @@ func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType str
 	md5Writer = md5.New()
 	md5Writer.Write(data)
 	etag2 := hex.EncodeToString(md5Writer.Sum(nil))
-	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum))
+	_, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum), opts)
 	if err != nil {
 		// Failed to upload object part, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())

cmd/object-handlers.go

@@ -89,7 +89,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	if api.CacheAPI() != nil {
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
-
+	opts := ObjectOptions{}
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
 		if getRequestAuthType(r) == authTypeAnonymous {
 			// As per "Permission" section in
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 				ConditionValues: getConditionValues(r, ""),
 				IsOwner:         false,
 			}) {
-				_, err := getObjectInfo(ctx, bucket, object)
+				_, err := getObjectInfo(ctx, bucket, object, opts)
 				if toAPIErrorCode(err) == ErrNoSuchKey {
 					s3Error = ErrNoSuchKey
 				}
@@ -125,7 +125,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		return
 	}
 
-	objInfo, err := getObjectInfo(ctx, bucket, object)
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -178,12 +178,15 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		writeErrorResponse(w, ErrInvalidQuoteFields, r.URL)
 		return
 	}
+	if len(selectReq.InputSerialization.CSV.RecordDelimiter) > 2 {
+		writeErrorResponse(w, ErrInvalidRequestParameter, r.URL)
+		return
+	}
 
 	getObject := objectAPI.GetObject
 	if api.CacheAPI() != nil && !crypto.SSEC.IsRequested(r.Header) {
 		getObject = api.CacheAPI().GetObject
 	}
-
 	reader, pipewriter := io.Pipe()
 
 	// Get the object.
@@ -208,7 +211,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	}
 	go func() {
 		defer reader.Close()
-		if gerr := getObject(ctx, bucket, object, 0, objInfo.Size, writer, objInfo.ETag); gerr != nil {
+		if gerr := getObject(ctx, bucket, object, 0, objInfo.Size, writer, objInfo.ETag, opts); gerr != nil {
 			pipewriter.CloseWithError(gerr)
 			return
 		}
@@ -222,9 +225,13 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	if selectReq.InputSerialization.CSV.FileHeaderInfo == "" {
 		selectReq.InputSerialization.CSV.FileHeaderInfo = CSVFileHeaderInfoNone
 	}
+	if selectReq.InputSerialization.CSV.RecordDelimiter == "" {
+		selectReq.InputSerialization.CSV.RecordDelimiter = "\n"
+	}
 	if selectReq.InputSerialization.CSV != nil {
 		options := &s3select.Options{
 			HasHeader:            selectReq.InputSerialization.CSV.FileHeaderInfo != CSVFileHeaderInfoNone,
+			RecordDelimiter:      selectReq.InputSerialization.CSV.RecordDelimiter,
 			FieldDelimiter:       selectReq.InputSerialization.CSV.FieldDelimiter,
 			Comments:             selectReq.InputSerialization.CSV.Comments,
 			Name:                 "S3Object", // Default table name for all objects
@@ -258,17 +265,20 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObject")
 
-	var object, bucket string
-	vars := mux.Vars(r)
-	bucket = vars["bucket"]
-	object = vars["object"]
-
-	// Fetch object stat info.
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
+	if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error
+		writeErrorResponse(w, ErrBadRequest, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+	opts := ObjectOptions{}
 
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
@@ -287,7 +297,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 				ConditionValues: getConditionValues(r, ""),
 				IsOwner:         false,
 			}) {
-				_, err := getObjectInfo(ctx, bucket, object)
+				_, err := getObjectInfo(ctx, bucket, object, opts)
 				if toAPIErrorCode(err) == ErrNoSuchKey {
 					s3Error = ErrNoSuchKey
 				}
@@ -297,15 +307,15 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	objInfo, err := getObjectInfo(ctx, bucket, object)
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
 
 	if objectAPI.IsEncryptionSupported() {
-		if apiErr, _ := DecryptObjectInfo(&objInfo, r.Header); apiErr != ErrNone {
-			writeErrorResponse(w, apiErr, r.URL)
+		if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		}
 	}
@@ -380,7 +390,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Reads the object at startOffset and writes to mw.
-	if err = getObject(ctx, bucket, object, startOffset, length, httpWriter, objInfo.ETag); err != nil {
+	if err = getObject(ctx, bucket, object, startOffset, length, httpWriter, objInfo.ETag, opts); err != nil {
 		if !httpWriter.HasWritten() && !statusCodeWritten { // write error response only if no data or headers has been written to client yet
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		}
@@ -403,13 +413,14 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 
 	// Notify object accessed via a GET request.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectAccessedGet,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectAccessedGet,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 }
 
@@ -419,22 +430,26 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HeadObject")
 
-	var object, bucket string
-	vars := mux.Vars(r)
-	bucket = vars["bucket"]
-	object = vars["object"]
-
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponseHeadersOnly(w, ErrServerNotInitialized)
 		return
 	}
+	if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error
+		writeErrorResponse(w, ErrBadRequest, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
 
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
 
+	opts := ObjectOptions{}
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone {
 		if getRequestAuthType(r) == authTypeAnonymous {
 			// As per "Permission" section in https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html
@@ -447,7 +462,7 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 				ConditionValues: getConditionValues(r, ""),
 				IsOwner:         false,
 			}) {
-				_, err := getObjectInfo(ctx, bucket, object)
+				_, err := getObjectInfo(ctx, bucket, object, opts)
 				if toAPIErrorCode(err) == ErrNoSuchKey {
 					s3Error = ErrNoSuchKey
 				}
@@ -457,15 +472,15 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	objInfo, err := getObjectInfo(ctx, bucket, object)
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
 		return
 	}
-
+	var encrypted bool
 	if objectAPI.IsEncryptionSupported() {
-		if apiErr, encrypted := DecryptObjectInfo(&objInfo, r.Header); apiErr != ErrNone {
-			writeErrorResponse(w, apiErr, r.URL)
+		if encrypted, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
+			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 			return
 		} else if encrypted {
 			s3Encrypted := crypto.S3.IsEncrypted(objInfo.UserDefined)
@@ -504,13 +519,14 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Notify object accessed via a HEAD request.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectAccessedHead,
-		BucketName: bucket,
-		Object:     objInfo,
-		ReqParams:  extractReqParams(r),
-		UserAgent:  r.UserAgent(),
-		Host:       host,
-		Port:       port,
+		EventName:    event.ObjectAccessedHead,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
 	})
 }
 
@@ -547,14 +563,19 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObject")
 
-	vars := mux.Vars(r)
-	dstBucket := vars["bucket"]
-	dstObject := vars["object"]
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
+	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
+		return
+	}
+
+	vars := mux.Vars(r)
+	dstBucket := vars["bucket"]
+	dstObject := vars["object"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
@@ -583,8 +604,9 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	var srcOpts, dstOpts ObjectOptions
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
-	srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject)
+	srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject, srcOpts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -592,7 +614,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject); err == nil {
+		if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts); err == nil {
 			writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
 			return
 		}
@@ -779,7 +801,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		var dstRecords []dns.SrvRecord
 		if dstRecords, err = globalDNSConfig.Get(dstBucket); err == nil {
 			go func() {
-				if gerr := objectAPI.GetObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag); gerr != nil {
+				if gerr := objectAPI.GetObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil {
 					pipeWriter.CloseWithError(gerr)
 					writeErrorResponse(w, ErrInternalError, r.URL)
 					return
@@ -796,7 +818,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 				writeErrorResponse(w, ErrInternalError, r.URL)
 				return
 			}
-			remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.Size, "", "", srcInfo.UserDefined)
+			remoteObjInfo, rerr := client.PutObject(dstBucket, dstObject, srcInfo.Reader, srcInfo.Size, "", "", srcInfo.UserDefined, dstOpts.ServerSideEncryption)
 			if rerr != nil {
 				pipeWriter.CloseWithError(rerr)
 				writeErrorResponse(w, ErrInternalError, r.URL)
@@ -808,7 +830,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	} else {
 		// Copy source object to destination, if source and destination
 		// object is same then only metadata is updated.
-		objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo)
+		objInfo, err = objectAPI.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 		if err != nil {
 			pipeWriter.CloseWithError(err)
 			writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -853,10 +875,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
-
-	// X-Amz-Copy-Source shouldn't be set for this call.
-	if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
-		writeErrorResponse(w, ErrInvalidCopySource, r.URL)
+	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
 		return
 	}
 
@@ -864,6 +884,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	bucket := vars["bucket"]
 	object := vars["object"]
 
+	// X-Amz-Copy-Source shouldn't be set for this call.
+	if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
+		writeErrorResponse(w, ErrInvalidCopySource, r.URL)
+		return
+	}
+
 	// Validate storage class metadata if present
 	if _, ok := r.Header[amzStorageClassCanonical]; ok {
 		if !isValidStorageClassMeta(r.Header.Get(amzStorageClassCanonical)) {
@@ -985,10 +1011,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
 	}
-
+	opts := ObjectOptions{}
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object); err == nil {
+		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
 			writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
 			return
 		}
@@ -1015,7 +1041,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Create the object..
-	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata)
+	objInfo, err := putObject(ctx, bucket, object, hashReader, metadata, opts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1058,16 +1084,20 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "NewMultipartUpload")
 
-	var object, bucket string
-	vars := mux.Vars(r)
-	bucket = vars["bucket"]
-	object = vars["object"]
-
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
+	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+	opts := ObjectOptions{}
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
@@ -1076,7 +1106,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err := objectAPI.GetObjectInfo(ctx, bucket, object); err == nil {
+		if _, err := objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
 			writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
 			return
 		}
@@ -1121,7 +1151,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	if api.CacheAPI() != nil && !hasServerSideEncryptionHeader(r.Header) {
 		newMultipartUpload = api.CacheAPI().NewMultipartUpload
 	}
-	uploadID, err := newMultipartUpload(ctx, bucket, object, metadata)
+	uploadID, err := newMultipartUpload(ctx, bucket, object, metadata, opts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1138,15 +1168,19 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CopyObjectPart")
 
-	vars := mux.Vars(r)
-	dstBucket := vars["bucket"]
-	dstObject := vars["object"]
-
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
+	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
+		return
+	}
+
+	vars := mux.Vars(r)
+	dstBucket := vars["bucket"]
+	dstObject := vars["object"]
 
 	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone {
 		writeErrorResponse(w, s3Error, r.URL)
@@ -1181,8 +1215,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		writeErrorResponse(w, ErrInvalidMaxParts, r.URL)
 		return
 	}
-
-	srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject)
+	var srcOpts, dstOpts ObjectOptions
+	srcInfo, err := objectAPI.GetObjectInfo(ctx, srcBucket, srcObject, srcOpts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1190,7 +1224,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject); err == nil {
+		if _, err = objectAPI.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts); err == nil {
 			writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
 			return
 		}
@@ -1315,11 +1349,10 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		}
 	}
 	srcInfo.Writer = writer
-
 	// Copy source object to destination, if source and destination
 	// object is same then only metadata is updated.
 	partInfo, err := objectAPI.CopyObjectPart(ctx, srcBucket, srcObject, dstBucket,
-		dstObject, uploadID, partID, startOffset, getLength, srcInfo)
+		dstObject, uploadID, partID, startOffset, getLength, srcInfo, srcOpts, dstOpts)
 	if err != nil {
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 		return
@@ -1339,14 +1372,19 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PutObjectPart")
 
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	object := vars["object"]
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(w, ErrServerNotInitialized, r.URL)
 		return
 	}
+	if !objectAPI.IsEncryptionSupported() && crypto.S3KMS.IsRequested(r.Header) {
+		writeErrorResponse(w, ErrNotImplemented, r.URL) // SSE-KMS is not supported
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
 
 	// X-Amz-Copy-Source shouldn't be set for this call.
 	if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
@@ -1460,9 +1498,10 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 
+	opts := ObjectOptions{}
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object); err == nil {
+		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
 			writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
 			return
 		}
@@ -1523,7 +1562,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	if api.CacheAPI() != nil && !hasServerSideEncryptionHeader(r.Header) {
 		putObjectPart = api.CacheAPI().PutObjectPart
 	}
-	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, hashReader)
+	partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, hashReader, opts)
 	if err != nil {
 		// Verify if the underlying error is signature mismatch.
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -1561,7 +1600,7 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err := objectAPI.GetObjectInfo(ctx, bucket, object); err == nil {
+		if _, err := objectAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err == nil {
 			writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
 			return
 		}
@@ -1636,7 +1675,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
-		if _, err := objectAPI.GetObjectInfo(ctx, bucket, object); err == nil {
+		if _, err := objectAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}); err == nil {
 			writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
 			return
 		}

cmd/object-handlers_test.go

@@ -77,7 +77,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData)
+		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -224,7 +224,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData)
+		_, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, ObjectOptions{})
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -746,13 +746,14 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
 		}
+		opts := ObjectOptions{}
 		if testCase.shouldPass {
 			// Verify whether the bucket obtained object is same as the one created.
 			if !bytes.Equal(testCase.expectedContent, actualContent) {
 				t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent))
 				continue
 			}
-			objInfo, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName)
+			objInfo, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName, opts)
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -764,7 +765,7 @@ func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketNam
 				t.Fatalf("Test %d: %s: ContentEncoding is set to \"%s\" which is unexpected, expected \"%s\"", i+1, instanceType, objInfo.ContentEncoding, expectedContentEncoding)
 			}
 			buffer := new(bytes.Buffer)
-			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer, objInfo.ETag)
+			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(testCase.dataLen), buffer, objInfo.ETag, opts)
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -787,6 +788,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 
 	var err error
 	objectName := "test-object"
+	opts := ObjectOptions{}
 	// byte data for PutObject.
 	bytesData := generateBytesData(6 * humanize.KiByte)
 
@@ -936,7 +938,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 			buffer := new(bytes.Buffer)
 
 			// Fetch the object to check whether the content is same as the one uploaded via PutObject.
-			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "")
+			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "", opts)
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -979,7 +981,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		if testCase.expectedRespStatus == http.StatusOK {
 			buffer := new(bytes.Buffer)
 			// Fetch the object to check whether the content is same as the one uploaded via PutObject.
-			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "")
+			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.objectName, 0, int64(len(bytesData)), buffer, "", opts)
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -1035,7 +1037,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 
 	objectName := "test-object"
 	var err error
-
+	opts := ObjectOptions{}
 	// set of byte data for PutObject.
 	// object has to be created before running tests for Copy Object.
 	// this is required even to assert the copied object,
@@ -1060,7 +1062,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	for i, input := range putObjectInputs {
 		// uploading the object.
 		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName,
-			mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData)
+			mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1073,7 +1075,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -1127,7 +1129,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	}
 
 	var buf bytes.Buffer
-	if err = obj.GetObject(context.Background(), bucketName, testObject, 0, int64(len(bytesData[0].byteData)), &buf, ""); err != nil {
+	if err = obj.GetObject(context.Background(), bucketName, testObject, 0, int64(len(bytesData[0].byteData)), &buf, "", opts); err != nil {
 		t.Fatalf("Test: %s reading completed file failed: <ERROR> %v", instanceType, err)
 	}
 	if !bytes.Equal(buf.Bytes(), bytesData[0].byteData) {
@@ -1146,7 +1148,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	objectName := "test-object"
 	var err error
-
+	opts := ObjectOptions{}
 	// set of byte data for PutObject.
 	// object has to be created before running tests for Copy Object.
 	// this is required even to assert the copied object,
@@ -1170,7 +1172,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1183,7 +1185,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -1474,7 +1476,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// object used for anonymous HTTP request test.
 	anonObject := "anon-object"
 	var err error
-
+	opts := ObjectOptions{}
 	// set of byte data for PutObject.
 	// object has to be created before running tests for Copy Object.
 	// this is required even to assert the copied object,
@@ -1507,7 +1509,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -1776,7 +1778,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		if rec.Code == http.StatusOK {
 			// See if the new object is formed.
 			// testing whether the copy was successful.
-			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "")
+			err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
 			if err != nil {
 				t.Fatalf("Test %d: %s: Failed to fetch the copied object: <ERROR> %s", i+1, instanceType, err)
 			}
@@ -2092,6 +2094,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 
 	var err error
 
+	var opts ObjectOptions
 	// object used for the test.
 	objectName := "test-object-new-multipart"
 
@@ -2102,7 +2105,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 
 	for i := 0; i < 2; i++ {
 		// initiate new multipart uploadID.
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -2144,7 +2147,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	// Iterating over creatPartCases to generate multipart chunks.
 	for _, part := range parts {
 		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID,
-			mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""))
+			mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err)
 		}
@@ -2440,7 +2443,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 	credentials auth.Credentials, t *testing.T) {
 
 	var err error
-
+	opts := ObjectOptions{}
 	// object used for the test.
 	objectName := "test-object-new-multipart"
 
@@ -2451,7 +2454,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	for i := 0; i < 2; i++ {
 		// initiate new multipart uploadID.
-		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil)
+		uploadID, err = obj.NewMultipartUpload(context.Background(), bucketName, objectName, nil, opts)
 		if err != nil {
 			// Failed to create NewMultipartUpload, abort.
 			t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -2493,7 +2496,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 	// Iterating over createPartCases to generate multipart chunks.
 	for _, part := range parts {
 		_, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID,
-			mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""))
+			mustGetHashReader(t, bytes.NewBufferString(part.inputReaderData), part.intputDataSize, part.inputMd5, ""), opts)
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err)
 		}
@@ -2603,7 +2606,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T) {
 
 	var err error
-
+	var opts ObjectOptions
 	objectName := "test-object"
 	// Object used for anonymous API request test.
 	anonObjectName := "test-anon-obj"
@@ -2631,7 +2634,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	// iterate through the above set of inputs and upload the object.
 	for i, input := range putObjectInputs {
 		// uploading the object.
-		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData)
+		_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetHashReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
 		// if object upload fails stop the test.
 		if err != nil {
 			t.Fatalf("Put Object case %d:  Error uploading object: <ERROR> %v", i+1, err)
@@ -2921,11 +2924,11 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 
 	// Initiate Multipart upload for testing PutObjectPartHandler.
 	testObject := "testobject"
-
+	var opts ObjectOptions
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -3324,11 +3327,11 @@ func TestAPIListObjectPartsHandler(t *testing.T) {
 func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
 	testObject := "testobject"
-
+	var opts ObjectOptions
 	// PutObjectPart API HTTP Handler has to be tested in isolation,
 	// that is without any other handler being registered,
 	// That's why NewMultipartUpload is initiated using ObjectLayer.
-	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, nil, opts)
 	if err != nil {
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("Minio %s : <ERROR>  %s", instanceType, err)
@@ -3337,7 +3340,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
 	uploadIDCopy := uploadID
 
 	// create an object Part, will be used to test list object parts.
-	_, err = obj.PutObjectPart(context.Background(), bucketName, testObject, uploadID, 1, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "5d41402abc4b2a76b9719d911017c592", ""))
+	_, err = obj.PutObjectPart(context.Background(), bucketName, testObject, uploadID, 1, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "5d41402abc4b2a76b9719d911017c592", ""), opts)
 	if err != nil {
 		t.Fatalf("Minio %s : %s.", instanceType, err)
 	}

cmd/object_api_suite_test.go

@@ -90,11 +90,12 @@ func (s *ObjectLayerAPISuite) TestMultipartObjectCreation(t *testing.T) {
 
 // Tests validate creation of part files during Multipart operation.
 func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	var opts ObjectOptions
 	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "")
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -105,7 +106,7 @@ func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErr
 		expectedETaghex := getMD5Hash(data)
 
 		var calcPartInfo PartInfo
-		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBuffer(data), int64(len(data)), expectedETaghex, ""))
+		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBuffer(data), int64(len(data)), expectedETaghex, ""), opts)
 		if err != nil {
 			t.Errorf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -133,11 +134,12 @@ func (s *ObjectLayerAPISuite) TestMultipartObjectAbort(t *testing.T) {
 
 // Tests validate abortion of Multipart operation.
 func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHandler) {
+	var opts ObjectOptions
 	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "")
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil)
+	uploadID, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -155,7 +157,7 @@ func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHan
 
 		metadata["md5"] = expectedETaghex
 		var calcPartInfo PartInfo
-		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, ""))
+		calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, ""), opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -179,6 +181,7 @@ func (s *ObjectLayerAPISuite) TestMultipleObjectCreation(t *testing.T) {
 func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	objects := make(map[string][]byte)
 	err := obj.MakeBucketWithLocation(context.Background(), "bucket", "")
+	var opts ObjectOptions
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -196,7 +199,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 		metadata := make(map[string]string)
 		metadata["etag"] = expectedETaghex
 		var objInfo ObjectInfo
-		objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata)
+		objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), metadata, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -207,7 +210,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 
 	for key, value := range objects {
 		var byteBuffer bytes.Buffer
-		err = obj.GetObject(context.Background(), "bucket", key, 0, int64(len(value)), &byteBuffer, "")
+		err = obj.GetObject(context.Background(), "bucket", key, 0, int64(len(value)), &byteBuffer, "", opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -215,7 +218,7 @@ func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrH
 			t.Errorf("%s: Mismatch of GetObject data with the expected one.", instanceType)
 		}
 
-		objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", key)
+		objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", key, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -246,10 +249,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
+	var opts ObjectOptions
 	// check before paging occurs.
 	for i := 0; i < 5; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
+		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -269,7 +273,7 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	// check after paging occurs pages work.
 	for i := 6; i <= 10; i++ {
 		key := "obj" + strconv.Itoa(i)
-		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
+		_, err = obj.PutObject(context.Background(), "bucket", key, mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -286,11 +290,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 	// check paging with prefix at end returns less objects.
 	{
-		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
+		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
-		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
+		_, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -328,11 +332,11 @@ func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) {
 
 	// check delimited results with delimiter and prefix.
 	{
-		_, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
+		_, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
-		_, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
+		_, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 		if err != nil {
 			t.Fatalf("%s: <ERROR> %s", instanceType, err)
 		}
@@ -441,22 +445,23 @@ func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHan
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
+	var opts ObjectOptions
 	uploadContent := "The list of parts was not in ascending order. The parts list must be specified in order by part number."
 	length := int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
 	uploadContent = "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
 	length = int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
 	var bytesBuffer bytes.Buffer
-	err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer, "")
+	err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer, "", opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -472,7 +477,8 @@ func (s *ObjectLayerAPISuite) TestNonExistantBucketOperations(t *testing.T) {
 
 // Tests validate that bucket operation on non-existent bucket fails.
 func testNonExistantBucketOperations(obj ObjectLayer, instanceType string, t TestErrHandler) {
-	_, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetHashReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), nil)
+	var opts ObjectOptions
+	_, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetHashReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), nil, opts)
 	if err == nil {
 		t.Fatal("Expected error but found nil")
 	}
@@ -519,11 +525,12 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	var bytesBuffer1 bytes.Buffer
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerEOF, length, "", ""), nil)
+	var opts ObjectOptions
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerEOF, length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer1, "")
+	err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer1, "", opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -532,11 +539,11 @@ func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	}
 
 	var bytesBuffer2 bytes.Buffer
-	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerNoEOF, length, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), "bucket", "object", mustGetHashReader(t, readerNoEOF, length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer2, "")
+	err = obj.GetObject(context.Background(), "bucket", "object", 0, length, &bytesBuffer2, "", opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -557,16 +564,17 @@ func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandle
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
+	var opts ObjectOptions
 	uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart
  upload might have been aborted or completed.`
 	length := int64(len(uploadContent))
-	_, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil)
+	_, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetHashReader(t, bytes.NewBufferString(uploadContent), length, "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
 	var bytesBuffer bytes.Buffer
-	err = obj.GetObject(context.Background(), "bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "")
+	err = obj.GetObject(context.Background(), "bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "", opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
@@ -703,7 +711,7 @@ func testNonExistantObjectInBucket(obj ObjectLayer, instanceType string, t TestE
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
 
-	_, err = obj.GetObjectInfo(context.Background(), "bucket", "dir1")
+	_, err = obj.GetObjectInfo(context.Background(), "bucket", "dir1", ObjectOptions{})
 	if err == nil {
 		t.Fatalf("%s: Expected error but found nil", instanceType)
 	}
@@ -732,7 +740,8 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
 	}
 	content := "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag."
 	length := int64(len(content))
-	_, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetHashReader(t, bytes.NewBufferString(content), length, "", ""), nil)
+	var opts ObjectOptions
+	_, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetHashReader(t, bytes.NewBufferString(content), length, "", ""), nil, opts)
 
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
@@ -753,7 +762,7 @@ func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string,
 	}
 
 	for i, testCase := range testCases {
-		_, expectedErr := obj.GetObjectInfo(context.Background(), bucketName, testCase.dir)
+		_, expectedErr := obj.GetObjectInfo(context.Background(), bucketName, testCase.dir, opts)
 		if expectedErr != nil && expectedErr.Error() != testCase.err.Error() {
 			t.Errorf("Test %d, %s: Expected error %s, got %s", i+1, instanceType, testCase.err, expectedErr)
 		}
@@ -771,13 +780,14 @@ func testContentType(obj ObjectLayer, instanceType string, t TestErrHandler) {
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
+	var opts ObjectOptions
 	uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed."
 	// Test empty.
-	_, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil)
+	_, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetHashReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}
-	objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", "minio.png")
+	objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", "minio.png", opts)
 	if err != nil {
 		t.Fatalf("%s: <ERROR> %s", instanceType, err)
 	}

cmd/posix.go

@@ -274,6 +274,10 @@ func (s *posix) String() string {
 	return s.diskPath
 }
 
+func (s *posix) LastError() error {
+	return nil
+}
+
 func (s *posix) Close() error {
 	close(s.stopUsageCh)
 	s.connected = false

cmd/post-policy_test.go

@@ -124,6 +124,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 	// get random bucket name.
 	bucketName := getRandomBucketName()
 
+	var opts ObjectOptions
 	// Register the API end points with XL/FS object layer.
 	apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"})
 
@@ -227,7 +228,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
 		}
 		// When the operation is successful, check if sending metadata is successful too
 		if rec.Code == http.StatusNoContent {
-			objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, testCase.objectName+"/upload.txt")
+			objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, testCase.objectName+"/upload.txt", opts)
 			if err != nil {
 				t.Error("Unexpected error: ", err)
 			}
@@ -427,6 +428,8 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
 	// Key specified in Form data
 	keyName := "test/object"
 
+	var opts ObjectOptions
+
 	// The final name of the upload object
 	targetObj := keyName + "/upload.txt"
 
@@ -478,7 +481,7 @@ func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t
 	}
 
 	// Get the uploaded object info
-	info, err := obj.GetObjectInfo(context.Background(), bucketName, targetObj)
+	info, err := obj.GetObjectInfo(context.Background(), bucketName, targetObj, opts)
 	if err != nil {
 		t.Error("Unexpected error: ", err)
 	}

cmd/prepare-storage.go

@@ -120,15 +120,33 @@ var errXLV3ThisEmpty = fmt.Errorf("XL format version 3 has This field empty")
 // connect to list of endpoints and load all XL disk formats, validate the formats are correct
 // and are in quorum, if no formats are found attempt to initialize all of them for the first
 // time. additionally make sure to close all the disks used in this attempt.
-func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, drivesPerSet int) (*formatXLV3, error) {
+func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointList, setCount, drivesPerSet int) (*formatXLV3, error) {
+	// Initialize all storage disks
 	storageDisks, err := initStorageDisks(endpoints)
 	if err != nil {
 		return nil, err
 	}
 	defer closeStorageDisks(storageDisks)
 
+	// Connect to all storage disks, a connection failure will be
+	// only logged after some retries.
+	for _, disk := range storageDisks {
+		if disk != nil {
+			connectErr := disk.LastError()
+			if connectErr != nil && retryCount >= 5 {
+				logger.Info("Unable to connect to %s: %v\n", disk.String(), connectErr.Error())
+			}
+		}
+	}
+
 	// Attempt to load all `format.json` from all disks.
 	formatConfigs, sErrs := loadFormatXLAll(storageDisks)
+	// Check if we have
+	for i, sErr := range sErrs {
+		if _, ok := formatCriticalErrors[sErr]; ok {
+			return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
+		}
+	}
 
 	// Pre-emptively check if one of the formatted disks
 	// is invalid. This function returns success for the
@@ -139,12 +157,6 @@ func connectLoadInitFormats(firstDisk bool, endpoints EndpointList, setCount, dr
 		return nil, err
 	}
 
-	for i, sErr := range sErrs {
-		if _, ok := formatCriticalErrors[sErr]; ok {
-			return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
-		}
-	}
-
 	// All disks report unformatted we should initialized everyone.
 	if shouldInitXLDisks(sErrs) && firstDisk {
 		return initFormatXL(context.Background(), storageDisks, setCount, drivesPerSet)
@@ -238,8 +250,8 @@ func waitForFormatXL(ctx context.Context, firstDisk bool, endpoints EndpointList
 	retryTimerCh := newRetryTimerSimple(doneCh)
 	for {
 		select {
-		case _ = <-retryTimerCh:
-			format, err := connectLoadInitFormats(firstDisk, endpoints, setCount, disksPerSet)
+		case retryCount := <-retryTimerCh:
+			format, err := connectLoadInitFormats(retryCount, firstDisk, endpoints, setCount, disksPerSet)
 			if err != nil {
 				switch err {
 				case errNotFirstDisk:

cmd/rpc/client.go

@@ -22,6 +22,8 @@ import (
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"reflect"
@@ -40,6 +42,28 @@ type Client struct {
 	serviceURL *xnet.URL
 }
 
+// closeResponse close non nil response with any response Body.
+// convenient wrapper to drain any remaining data on response body.
+//
+// Subsequently this allows golang http RoundTripper
+// to re-use the same connection for future requests.
+func closeResponse(body io.ReadCloser) {
+	// Callers should close resp.Body when done reading from it.
+	// If resp.Body is not closed, the Client's underlying RoundTripper
+	// (typically Transport) may not be able to re-use a persistent TCP
+	// connection to the server for a subsequent "keep-alive" request.
+	if body != nil {
+		// Drain any remaining Body and then close the connection.
+		// Without this closing connection would disallow re-using
+		// the same connection for future uses.
+		//  - http://stackoverflow.com/a/17961593/4465767
+		bufp := b512pool.Get().(*[]byte)
+		defer b512pool.Put(bufp)
+		io.CopyBuffer(ioutil.Discard, body, *bufp)
+		body.Close()
+	}
+}
+
 // Call - calls service method on RPC server.
 func (client *Client) Call(serviceMethod string, args, reply interface{}) error {
 	replyKind := reflect.TypeOf(reply).Kind()
@@ -69,7 +93,7 @@ func (client *Client) Call(serviceMethod string, args, reply interface{}) error
 	if err != nil {
 		return err
 	}
-	defer response.Body.Close()
+	defer closeResponse(response.Body)
 
 	if response.StatusCode != http.StatusOK {
 		return fmt.Errorf("%v rpc call failed with error code %v", serviceMethod, response.StatusCode)

cmd/rpc/pool.go

@@ -21,6 +21,13 @@ import (
 	"sync"
 )
 
+var b512pool = sync.Pool{
+	New: func() interface{} {
+		buf := make([]byte, 512)
+		return &buf
+	},
+}
+
 // A Pool is a type-safe wrapper around a sync.Pool.
 type Pool struct {
 	p *sync.Pool

cmd/server-main.go

@@ -91,7 +91,7 @@ ENVIRONMENT VARIABLES:
      MINIO_DOMAIN:    To enable bucket DNS requests, set this value to Minio host domain name.
      MINIO_PUBLIC_IPS: To enable bucket DNS requests, set this value to list of Minio host public IP(s) delimited by ",".
      MINIO_ETCD_ENDPOINTS: To enable bucket DNS requests, set this value to list of etcd endpoints delimited by ",".
-	
+
    KMS:
      MINIO_SSE_VAULT_ENDPOINT: To enable Vault as KMS,set this value to Vault endpoint.
      MINIO_SSE_VAULT_APPROLE_ID: To enable Vault as KMS,set this value to Vault AppRole ID.
@@ -123,7 +123,7 @@ EXAMPLES:
      $ export MINIO_CACHE_EXPIRY=40
      $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}} /home/shared
-	
+
   7. Start minio server with KMS enabled.
      $ export MINIO_SSE_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
      $ export MINIO_SSE_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
@@ -224,11 +224,6 @@ func serverMain(ctx *cli.Context) {
 	// Handle all server environment vars.
 	serverHandleEnvVars()
 
-	// In distributed setup users need to set ENVs always.
-	if !globalIsEnvCreds && globalIsDistXL {
-		logger.Fatal(uiErrEnvCredentialsMissingServer(nil), "Unable to initialize minio server in distributed mode")
-	}
-
 	// Create certs path.
 	logger.FatalIf(createConfigDir(), "Unable to initialize configuration files")
 
@@ -258,9 +253,33 @@ func serverMain(ctx *cli.Context) {
 		checkUpdate(mode)
 	}
 
-	// Enforce ENV credentials for distributed setup such that we can create the first config.
-	if globalIsDistXL && !globalIsEnvCreds {
-		logger.Fatal(uiErrInvalidCredentials(nil), "Unable to start the server in distrbuted mode. In distributed mode we require explicit credentials.")
+	// FIXME: This code should be removed in future releases and we should have mandatory
+	// check for ENVs credentials under distributed setup. Until all users migrate we
+	// are intentionally providing backward compatibility.
+	{
+		// Check for backward compatibility and newer style.
+		if !globalIsEnvCreds && globalIsDistXL {
+			// Try to load old config file if any, for backward compatibility.
+			var config = &serverConfig{}
+			if _, err = Load(getConfigFile(), config); err == nil {
+				globalActiveCred = config.Credential
+			}
+
+			if os.IsNotExist(err) {
+				if _, err = Load(getConfigFile()+".deprecated", config); err == nil {
+					globalActiveCred = config.Credential
+				}
+			}
+
+			if globalActiveCred.IsValid() {
+				// Credential is valid don't throw an error instead print a message regarding deprecation of 'config.json'
+				// based model and proceed to use it for now in distributed setup.
+				logger.Info(`Supplying credentials from your 'config.json' is **DEPRECATED**, Access key and Secret key in distributed server mode is expected to be specified with environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY. This approach will become mandatory in future releases, please migrate to this approach soon.`)
+			} else {
+				// Credential is not available anywhere by both means, we cannot start distributed setup anymore, fail eagerly.
+				logger.Fatal(uiErrEnvCredentialsMissingDistributed(nil), "Unable to initialize the server in distributed mode")
+			}
+		}
 	}
 
 	// Set system resources to maximum.
@@ -313,17 +332,21 @@ func serverMain(ctx *cli.Context) {
 		logger.FatalIf(err, "Unable to initialize backend")
 	}
 
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = newObject
-	globalObjLayerMutex.Unlock()
-
 	// Populate existing buckets to the etcd backend
 	if globalDNSConfig != nil {
 		initFederatorBackend(newObject)
 	}
 
-	// Initialize server config.
-	initConfig()
+	// Re-enable logging
+	logger.Disable = false
+
+	// Create a new config system.
+	globalConfigSys = NewConfigSys()
+
+	// Initialize config system.
+	if err = globalConfigSys.Init(newObject); err != nil {
+		logger.Fatal(err, "Unable to initialize config system")
+	}
 
 	// Load logger subsystem
 	loadLoggers()
@@ -335,22 +358,11 @@ func serverMain(ctx *cli.Context) {
 		logger.FatalIf(err, "Unable to initialize disk caching")
 	}
 
-	// Re-enable logging
-	logger.Disable = false
-
-	// Create a new config system.
-	globalConfigSys = NewConfigSys()
-
-	// Initialize config system.
-	if err := globalConfigSys.Init(newObjectLayerFn()); err != nil {
-		logger.Fatal(err, "Unable to initialize config system")
-	}
-
 	// Create new policy system.
 	globalPolicySys = NewPolicySys()
 
 	// Initialize policy system.
-	if err := globalPolicySys.Init(newObjectLayerFn()); err != nil {
+	if err := globalPolicySys.Init(newObject); err != nil {
 		logger.Fatal(err, "Unable to initialize policy system")
 	}
 
@@ -358,10 +370,14 @@ func serverMain(ctx *cli.Context) {
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 
 	// Initialize notification system.
-	if err := globalNotificationSys.Init(newObjectLayerFn()); err != nil {
+	if err := globalNotificationSys.Init(newObject); err != nil {
 		logger.Fatal(err, "Unable to initialize notification system")
 	}
 
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = newObject
+	globalObjLayerMutex.Unlock()
+
 	// Prints the formatted startup message once object layer is initialized.
 	apiEndpoints := getAPIEndpoints(globalMinioAddr)
 	printStartupMessage(apiEndpoints)

cmd/server-startup-msg.go

@@ -174,7 +174,7 @@ func printObjectAPIMsg() {
 // Get formatted disk/storage info message.
 func getStorageInfoMsg(storageInfo StorageInfo) string {
 	var msg string
-	if storageInfo.Backend.Type == Erasure {
+	if storageInfo.Backend.Type == BackendErasure {
 		diskInfo := fmt.Sprintf(" %d Online, %d Offline. ", storageInfo.Backend.OnlineDisks, storageInfo.Backend.OfflineDisks)
 		msg += colorBlue("Status:") + fmt.Sprintf(getFormatStr(len(diskInfo), 8), diskInfo)
 	}

cmd/server-startup-msg_test.go

@@ -30,7 +30,7 @@ import (
 // Tests if we generate storage info.
 func TestStorageInfoMsg(t *testing.T) {
 	infoStorage := StorageInfo{}
-	infoStorage.Backend.Type = Erasure
+	infoStorage.Backend.Type = BackendErasure
 	infoStorage.Backend.OnlineDisks = 7
 	infoStorage.Backend.OfflineDisks = 1
 

cmd/storage-class_test.go

@@ -175,6 +175,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	resetGlobalStorageEnvs()
 	bucket := getRandomBucketName()
 
+	var opts ObjectOptions
 	// make data with more than one part
 	partCount := 3
 	data := bytes.Repeat([]byte("a"), int(globalPutPartSize)*partCount)
@@ -188,7 +189,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 
 	// Object for test case 1 - No StorageClass defined, no MetaData in PutObject
 	object1 := "object1"
-	_, err = obj.PutObject(context.Background(), bucket, object1, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil)
+	_, err = obj.PutObject(context.Background(), bucket, object1, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), nil, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -199,7 +200,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object2 := "object2"
 	metadata2 := make(map[string]string)
 	metadata2["x-amz-storage-class"] = reducedRedundancyStorageClass
-	_, err = obj.PutObject(context.Background(), bucket, object2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata2)
+	_, err = obj.PutObject(context.Background(), bucket, object2, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata2, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -210,7 +211,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object3 := "object3"
 	metadata3 := make(map[string]string)
 	metadata3["x-amz-storage-class"] = standardStorageClass
-	_, err = obj.PutObject(context.Background(), bucket, object3, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata3)
+	_, err = obj.PutObject(context.Background(), bucket, object3, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata3, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -226,7 +227,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object4, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata4)
+	_, err = obj.PutObject(context.Background(), bucket, object4, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata4, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -244,7 +245,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object5, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata5)
+	_, err = obj.PutObject(context.Background(), bucket, object5, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata5, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -262,7 +263,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object6, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata6)
+	_, err = obj.PutObject(context.Background(), bucket, object6, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata6, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}
@@ -280,7 +281,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		Scheme: "EC",
 	}
 
-	_, err = obj.PutObject(context.Background(), bucket, object7, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata7)
+	_, err = obj.PutObject(context.Background(), bucket, object7, mustGetHashReader(t, bytes.NewReader(data), int64(len(data)), "", ""), metadata7, opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
 	}

cmd/storage-interface.go

@@ -27,7 +27,9 @@ type StorageAPI interface {
 
 	// Storage operations.
 	IsOnline() bool // Returns true if disk is online.
+	LastError() error
 	Close() error
+
 	DiskInfo() (info DiskInfo, err error)
 
 	// Volume operations.

cmd/storage-rpc-client.go

@@ -104,6 +104,8 @@ func toStorageErr(err error) error {
 type StorageRPCClient struct {
 	*RPCClient
 	connected bool
+	// Plain error of the last RPC call
+	lastRPCError error
 }
 
 // Stringer provides a canonicalized representation of network device.
@@ -114,6 +116,11 @@ func (client *StorageRPCClient) String() string {
 	return url.String()
 }
 
+// LastError - returns the last RPC call result, nil or error if any
+func (client *StorageRPCClient) LastError() error {
+	return client.lastRPCError
+}
+
 // Close - closes underneath RPC client.
 func (client *StorageRPCClient) Close() error {
 	client.connected = false
@@ -125,14 +132,22 @@ func (client *StorageRPCClient) IsOnline() bool {
 	return client.connected
 }
 
+func (client *StorageRPCClient) connect() {
+	err := client.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{})
+	client.lastRPCError = err
+	client.connected = err == nil
+}
+
 func (client *StorageRPCClient) call(handler string, args interface {
 	SetAuthArgs(args AuthArgs)
 }, reply interface{}) error {
+
 	if !client.connected {
 		return errDiskNotFound
 	}
 
 	err := client.Call(handler, args, reply)
+	client.lastRPCError = err
 	if err == nil {
 		return nil
 	}
@@ -227,7 +242,7 @@ func (client *StorageRPCClient) ReadFile(volume string, path string, offset int6
 		Vol:      volume,
 		Path:     path,
 		Offset:   offset,
-		Buffer:   buffer,
+		Length:   int64(len(buffer)),
 		Verified: true, // mark read as verified by default
 	}
 	if verifier != nil {
@@ -318,6 +333,7 @@ func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
 	logger.FatalIf(err, "Unable to parse storage RPC Host", context.Background())
 	rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
 	logger.FatalIf(err, "Unable to initialize storage RPC client", context.Background())
-	rpcClient.connected = rpcClient.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{}) == nil
+	// Attempt first try connection and save error if any.
+	rpcClient.connect()
 	return rpcClient
 }

cmd/storage-rpc-server.go

@@ -127,7 +127,7 @@ type ReadFileArgs struct {
 	Vol          string
 	Path         string
 	Offset       int64
-	Buffer       []byte
+	Length       int64
 	Algo         BitrotAlgorithm
 	ExpectedHash []byte
 	Verified     bool
@@ -140,13 +140,14 @@ func (receiver *storageRPCReceiver) ReadFile(args *ReadFileArgs, reply *[]byte)
 		verifier = NewBitrotVerifier(args.Algo, args.ExpectedHash)
 	}
 
-	n, err := receiver.local.ReadFile(args.Vol, args.Path, args.Offset, args.Buffer, verifier)
+	buf := make([]byte, args.Length)
+	n, err := receiver.local.ReadFile(args.Vol, args.Path, args.Offset, buf, verifier)
 	// Ignore io.ErrEnexpectedEOF for short reads i.e. less content available than requested.
 	if err == io.ErrUnexpectedEOF {
 		err = nil
 	}
 
-	*reply = args.Buffer[0:n]
+	*reply = buf[0:n]
 	return err
 }
 

cmd/storage-rpc_test.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -355,7 +354,6 @@ func testStorageAPIReadFile(t *testing.T, storage StorageAPI) {
 		expectErr := (err != nil)
 
 		if expectErr != testCase.expectErr {
-			fmt.Println(err)
 			t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
 		}