xl: Fix typo in PutObjectPart when part size is 10Mb (#6574)

PutObjectPart forgot to allocate buffer memory when the size
of the uploaded part is exactly equal to blockSizeV1 = 10 Mb.

Dockerfile.dev

@@ -1,27 +1,20 @@
-FROM golang:1.10.1-alpine3.7
+FROM alpine:3.7
 
 LABEL maintainer="Minio Inc <dev@minio.io>"
 
-ENV GOPATH /go
-ENV PATH $PATH:$GOPATH/bin
-ENV CGO_ENABLED 0
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
+COPY minio /usr/bin/
+
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key
 
-WORKDIR /go/src/github.com/minio/
-
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
-
-COPY . /go/src/github.com/minio/minio
-
-RUN  \
+RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' && \
-     apk add --no-cache --virtual .build-deps git && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
-     cd /go/src/github.com/minio/minio && \
-     go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
+     chmod +x /usr/bin/minio  && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/healthcheck.sh
 
 EXPOSE 9000
 

Makefile

@@ -75,6 +75,9 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
 
+docker: build
+	@docker build -t $(TAG) . -f Dockerfile.dev
+
 pkg-add:
 	@echo "Adding new package $(PKG)"
 	@${GOPATH}/bin/govendor add $(PKG)

browser/app/js/alert/actions.js

@@ -22,7 +22,7 @@ export let alertId = 0
 export const set = alert => {
   const id = alertId++
   return (dispatch, getState) => {
-    if (alert.type !== "danger") {
+    if (alert.type !== "danger" || alert.autoClear) {
       setTimeout(() => {
         dispatch({
           type: CLEAR,

browser/app/js/buckets/BucketList.js

@@ -67,8 +67,12 @@ const mapDispatchToProps = dispatch => {
   return {
     fetchBuckets: () => dispatch(actionsBuckets.fetchBuckets()),
     setBucketList: buckets => dispatch(actionsBuckets.setList(buckets)),
-    selectBucket: bucket => dispatch(actionsBuckets.selectBucket(bucket))
+    selectBucket: (bucket, prefix) =>
+      dispatch(actionsBuckets.selectBucket(bucket, prefix))
   }
 }
 
-export default connect(mapStateToProps, mapDispatchToProps)(BucketList)
+export default connect(
+  mapStateToProps,
+  mapDispatchToProps
+)(BucketList)

browser/app/js/objects/__tests__/actions.test.js

@@ -123,9 +123,6 @@ describe("Objects actions", () => {
       objects: { currentPrefix: "" }
     })
     const expectedActions = [
-      {
-        type: "alert/CLEAR"
-      },
       {
         type: "objects/SET_LIST",
         objects: [{ name: "test1" }, { name: "test2" }],
@@ -157,9 +154,6 @@ describe("Objects actions", () => {
       objects: { currentPrefix: "" }
     })
     const expectedActions = [
-      {
-        type: "alert/CLEAR"
-      },
       {
         type: "objects/APPEND_LIST",
         objects: [{ name: "test1" }, { name: "test2" }],
@@ -183,15 +177,13 @@ describe("Objects actions", () => {
       objects: { currentPrefix: "" }
     })
     const expectedActions = [
-      {
-        type: "alert/CLEAR"
-      },
       {
         type: "alert/SET",
         alert: {
           type: "danger",
           message: "listobjects is denied",
-          id: alertActions.alertId
+          id: alertActions.alertId,
+          autoClear: true
         }
       },
       {
@@ -252,7 +244,6 @@ describe("Objects actions", () => {
     })
     const expectedActions = [
       { type: "objects/SET_CURRENT_PREFIX", prefix: "abc/" },
-      { type: "alert/CLEAR" },
       { type: "objects/CHECKED_LIST_RESET" }
     ]
     store.dispatch(actionsObjects.selectPrefix("abc/"))

browser/app/js/objects/actions.js

@@ -64,43 +64,47 @@ export const fetchObjects = append => {
       buckets: { currentBucket },
       objects: { currentPrefix, marker }
     } = getState()
-    dispatch(alertActions.clear())
     if (currentBucket) {
       return web
-      .ListObjects({
-        bucketName: currentBucket,
-        prefix: currentPrefix,
-        marker: append ? marker : ""
-      })
-      .then(res => {
-        let objects = []
-        if (res.objects) {
-          objects = res.objects.map(object => {
-            return {
-              ...object,
-              name: object.name.replace(currentPrefix, "")
-            }
-          })
-        }
-        if (append) {
-          dispatch(appendList(objects, res.nextmarker, res.istruncated))
-        } else {
-          dispatch(setList(objects, res.nextmarker, res.istruncated))
-          dispatch(setSortBy(""))
-          dispatch(setSortOrder(false))
-        }
-        dispatch(setPrefixWritable(res.writable))
-      })
-      .catch(err => {
-        if (web.LoggedIn()) {
-          dispatch(alertActions.set({ type: "danger", message: err.message }))
-          dispatch(resetList())
-        }
-        else {
-          history.push("/login")
-        }
-      })
-    } 
+        .ListObjects({
+          bucketName: currentBucket,
+          prefix: currentPrefix,
+          marker: append ? marker : ""
+        })
+        .then(res => {
+          let objects = []
+          if (res.objects) {
+            objects = res.objects.map(object => {
+              return {
+                ...object,
+                name: object.name.replace(currentPrefix, "")
+              }
+            })
+          }
+          if (append) {
+            dispatch(appendList(objects, res.nextmarker, res.istruncated))
+          } else {
+            dispatch(setList(objects, res.nextmarker, res.istruncated))
+            dispatch(setSortBy(""))
+            dispatch(setSortOrder(false))
+          }
+          dispatch(setPrefixWritable(res.writable))
+        })
+        .catch(err => {
+          if (web.LoggedIn()) {
+            dispatch(
+              alertActions.set({
+                type: "danger",
+                message: err.message,
+                autoClear: true
+              })
+            )
+            dispatch(resetList())
+          } else {
+            history.push("/login")
+          }
+        })
+    }
   }
 }
 

cmd/admin-handlers.go

@@ -17,13 +17,16 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
+	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -275,6 +278,127 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
+// StartProfilingResult contains the status of the starting
+// profiling action in a given server
+type StartProfilingResult struct {
+	NodeName string `json:"nodeName"`
+	Success  bool   `json:"success"`
+	Error    string `json:"error"`
+}
+
+// StartProfilingHandler - POST /minio/admin/v1/profiling/start?profilerType={profilerType}
+// ----------
+// Enable server profiling
+func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	profiler := vars["profilerType"]
+
+	startProfilingResult := make([]StartProfilingResult, len(globalAdminPeers))
+
+	// Call StartProfiling function on all nodes and save results
+	wg := sync.WaitGroup{}
+	for i, peer := range globalAdminPeers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			result := StartProfilingResult{NodeName: peer.addr}
+			if err := peer.cmdRunner.StartProfiling(profiler); err != nil {
+				result.Error = err.Error()
+				return
+			}
+			result.Success = true
+			startProfilingResult[idx] = result
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Create JSON result and send it to the client
+	startProfilingResultInBytes, err := json.Marshal(startProfilingResult)
+	if err != nil {
+		writeCustomErrorResponseJSON(w, http.StatusInternalServerError, err.Error(), r.URL)
+		return
+	}
+	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
+}
+
+// dummyFileInfo represents a dummy representation of a profile data file
+// present only in memory, it helps to generate the zip stream.
+type dummyFileInfo struct {
+	name    string
+	size    int64
+	mode    os.FileMode
+	modTime time.Time
+	isDir   bool
+	sys     interface{}
+}
+
+func (f dummyFileInfo) Name() string       { return f.name }
+func (f dummyFileInfo) Size() int64        { return f.size }
+func (f dummyFileInfo) Mode() os.FileMode  { return f.mode }
+func (f dummyFileInfo) ModTime() time.Time { return f.modTime }
+func (f dummyFileInfo) IsDir() bool        { return f.isDir }
+func (f dummyFileInfo) Sys() interface{}   { return f.sys }
+
+// DownloadProfilingHandler - POST /minio/admin/v1/profiling/download
+// ----------
+// Download profiling information of all nodes in a zip format
+func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
+	adminAPIErr := checkAdminRequestAuthType(r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	profilingDataFound := false
+
+	// Initialize a zip writer which will provide a zipped content
+	// of profiling data of all nodes
+	zipWriter := zip.NewWriter(w)
+	defer zipWriter.Close()
+
+	for i, peer := range globalAdminPeers {
+		// Get profiling data from a node
+		data, err := peer.cmdRunner.DownloadProfilingData()
+		if err != nil {
+			logger.LogIf(context.Background(), fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
+			continue
+		}
+
+		profilingDataFound = true
+
+		// Send profiling data to zip as file
+		header, err := zip.FileInfoHeader(dummyFileInfo{
+			name:    fmt.Sprintf("profiling-%d", i),
+			size:    int64(len(data)),
+			mode:    0600,
+			modTime: time.Now().UTC(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if err != nil {
+			continue
+		}
+		writer, err := zipWriter.CreateHeader(header)
+		if err != nil {
+			continue
+		}
+		if _, err = io.Copy(writer, bytes.NewBuffer(data)); err != nil {
+			return
+		}
+	}
+
+	if !profilingDataFound {
+		writeErrorResponseJSON(w, ErrAdminProfilerNotEnabled, r.URL)
+		return
+	}
+}
+
 // extractHealInitParams - Validates params for heal init API.
 func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
 	hs madmin.HealOpts, clientToken string, forceStart bool,
@@ -551,7 +675,7 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 			continue
 		}
 		val := gjson.Get(configStr, key)
-		if j, err := sjson.Set(newConfigStr, normalizeJSONKey(key), val.Value()); err == nil {
+		if j, ierr := sjson.Set(newConfigStr, normalizeJSONKey(key), val.Value()); ierr == nil {
 			newConfigStr = j
 		}
 	}
@@ -663,8 +787,6 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Reply to the client before restarting minio server.
 	writeSuccessResponseHeadersOnly(w)
-
-	sendServiceCmd(globalAdminPeers, serviceRestart)
 }
 
 func convertValueType(elem []byte, jsonType gjson.Type) (interface{}, error) {
@@ -796,8 +918,6 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 
 	// Send success response
 	writeSuccessResponseHeadersOnly(w)
-
-	sendServiceCmd(globalAdminPeers, serviceRestart)
 }
 
 // UpdateCredsHandler - POST /minio/admin/v1/config/credential

cmd/admin-handlers_test.go

@@ -27,6 +27,7 @@ import (
 	"net/http/httptest"
 	"net/url"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -37,15 +38,13 @@ import (
 
 var (
 	configJSON = []byte(`{
-	"version": "28",
+	"version": "30",
 	"credential": {
 		"accessKey": "minio",
 		"secretKey": "minio123"
 	},
 	"region": "",
-	"browser": "on",
 	"worm": "off",
-	"domain": "",
 	"storageclass": {
 		"standard": "",
 		"rrs": ""
@@ -187,20 +186,24 @@ var (
 				"endpoint": ""
 			}
 		}
-	    },
-	    "logger": {
+	},
+	"logger": {
 		"console": {
-		    "enabled": true
+			"enabled": true
 		},
 		"http": {
-		    "1": {
-			"enabled": false,
-			"endpoint": "http://user:example@localhost:9001/api/endpoint"
-		    }
+			"target1": {
+				"enabled": false,
+				"endpoint": "https://username:password@example.com/api"
+			}
 		}
-	    }
-
-	}`)
+	},
+	"compress": {
+		"enabled": false,
+        	"extensions":[".txt",".log",".csv",".json"],
+        	"mime-types":["text/csv","text/plain","application/json"]
+	}
+}`)
 )
 
 // adminXLTestBed - encapsulates subsystems that need to be setup for
@@ -508,10 +511,16 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
+	var wg sync.WaitGroup
+
 	// Setting up a go routine to simulate ServerRouter's
 	// handleServiceSignals for stop and restart commands.
 	if cmd == restartCmd {
-		go testServiceSignalReceiver(cmd, t)
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			testServiceSignalReceiver(cmd, t)
+		}()
 	}
 	credentials := globalServerConfig.GetCredential()
 
@@ -547,6 +556,9 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
 			http.StatusOK, rec.Code, string(resp))
 	}
+
+	// Wait until testServiceSignalReceiver() called in a goroutine quits.
+	wg.Wait()
 }
 
 // Test for service status management REST API.
@@ -701,10 +713,6 @@ func TestSetConfigHandler(t *testing.T) {
 	globalMinioAddr = "127.0.0.1:9000"
 	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
-	// SetConfigHandler restarts minio setup - need to start a
-	// signal receiver to receive on globalServiceSignalCh.
-	go testServiceSignalReceiver(restartCmd, t)
-
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}
 	queryVal.Set("config", "")

cmd/admin-heal-ops.go

@@ -20,12 +20,14 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"runtime"
 	"strings"
 	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // healStatusSummary - overall short summary of a healing sequence
@@ -383,7 +385,6 @@ func (h *healSequence) stop() {
 // sequence automatically resumes. The return value indicates if the
 // operation succeeded.
 func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
-
 	// start a timer to keep an upper time limit to find an empty
 	// slot to add the given heal result - if no slot is found it
 	// means that the server is holding the maximum amount of
@@ -519,6 +520,9 @@ func (h *healSequence) traverseAndHeal() {
 	// Start with format healing
 	checkErr(h.healDiskFormat)
 
+	// Start healing the config.
+	checkErr(h.healConfig)
+
 	// Heal buckets and objects
 	checkErr(h.healBuckets)
 
@@ -529,9 +533,72 @@ func (h *healSequence) traverseAndHeal() {
 	close(h.traverseAndHealDoneCh)
 }
 
+// healConfig - heals config.json, retrun value indicates if a failure occurred.
+func (h *healSequence) healConfig() error {
+	// Get current object layer instance.
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// NOTE: Healing on configs is run regardless
+	// of any bucket being selected, this is to ensure that
+	// configs are always uptodate and correct.
+	marker := ""
+	isTruncated := true
+	for isTruncated {
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to heal
+			waitCount := 60
+			// Any requests in progress, delay the heal.
+			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
+		}
+
+		// Lists all objects under `config` prefix.
+		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, minioConfigPrefix,
+			marker, "", 1000)
+		if err != nil {
+			return errFnHealFromAPIErr(err)
+		}
+
+		for index := range objectInfos.Objects {
+			if h.isQuitting() {
+				return errHealStopSignalled
+			}
+			o := objectInfos.Objects[index]
+			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun)
+			// Object might have been deleted, by the time heal
+			// was attempted we ignore this file an move on.
+			if isErrObjectNotFound(herr) {
+				continue
+			}
+			if herr != nil {
+				return herr
+			}
+			res.Type = madmin.HealItemBucketMetadata
+			if err = h.pushHealResultItem(res); err != nil {
+				return err
+			}
+		}
+
+		isTruncated = objectInfos.IsTruncated
+		marker = objectInfos.NextMarker
+	}
+
+	return nil
+}
+
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func (h *healSequence) healDiskFormat() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -557,6 +624,10 @@ func (h *healSequence) healDiskFormat() error {
 
 // healBuckets - check for all buckets heal or just particular bucket.
 func (h *healSequence) healBuckets() error {
+	if h.isQuitting() {
+		return errHealStopSignalled
+	}
+
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
 		return h.healBucket(h.bucket)
@@ -584,10 +655,6 @@ func (h *healSequence) healBuckets() error {
 
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string) error {
-	if h.isQuitting() {
-		return errHealStopSignalled
-	}
-
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -612,8 +679,7 @@ func (h *healSequence) healBucket(bucket string) error {
 			// and if so heal it.
 			_, err = objectAPI.GetObjectInfo(h.ctx, bucket, h.objPrefix, ObjectOptions{})
 			if err == nil {
-				err = h.healObject(bucket, h.objPrefix)
-				if err != nil {
+				if err = h.healObject(bucket, h.objPrefix); err != nil {
 					return err
 				}
 			}
@@ -622,17 +688,40 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
+	entries := runtime.NumCPU() * globalEndpoints.Nodes()
+
 	marker := ""
 	isTruncated := true
 	for isTruncated {
+		if globalHTTPServer != nil {
+			// Wait at max 1 minute for an inprogress request
+			// before proceeding to heal
+			waitCount := 60
+			// Any requests in progress, delay the heal.
+			for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
+				waitCount--
+				time.Sleep(1 * time.Second)
+			}
+		}
+
+		// Heal numCPU * nodes objects at a time.
 		objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
-			h.objPrefix, marker, "", 1000)
+			h.objPrefix, marker, "", entries)
 		if err != nil {
 			return errFnHealFromAPIErr(err)
 		}
 
-		for _, o := range objectInfos.Objects {
-			if err := h.healObject(o.Bucket, o.Name); err != nil {
+		g := errgroup.WithNErrs(len(objectInfos.Objects))
+		for index := range objectInfos.Objects {
+			index := index
+			g.Go(func() error {
+				o := objectInfos.Objects[index]
+				return h.healObject(o.Bucket, o.Name)
+			}, index)
+		}
+
+		for _, err := range g.Wait() {
+			if err != nil {
 				return err
 			}
 		}
@@ -656,6 +745,9 @@ func (h *healSequence) healObject(bucket, object string) error {
 	}
 
 	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	if isErrObjectNotFound(err) {
+		return nil
+	}
 	if err != nil {
 		hri.Detail = err.Error()
 	}

cmd/admin-router.go

@@ -60,6 +60,11 @@ func registerAdminRouter(router *mux.Router) {
 	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 	adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
 
+	// Profiling operations
+	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+		Queries("profilerType", "{profilerType:.*}")
+	adminV1Router.Methods(http.MethodGet).Path("/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+
 	/// Config operations
 
 	// Update credentials

cmd/admin-rpc-client.go

@@ -68,6 +68,22 @@ func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
 	return reply, err
 }
 
+// StartProfiling - starts profiling in the remote server.
+func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
+	args := StartProfilingArgs{Profiler: profiler}
+	reply := VoidReply{}
+	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
+}
+
+// DownloadProfilingData - returns profiling data of the remote server.
+func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+
+	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
+	return reply, err
+}
+
 // NewAdminRPCClient - returns new admin RPC client.
 func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
 	scheme := "http"
@@ -112,6 +128,8 @@ type adminCmdRunner interface {
 	ReInitFormat(dryRun bool) error
 	ServerInfo() (ServerInfoData, error)
 	GetConfig() ([]byte, error)
+	StartProfiling(string) error
+	DownloadProfilingData() ([]byte, error)
 }
 
 // adminPeer - represents an entity that implements admin API RPCs.
@@ -139,9 +157,9 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
 
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
+		logger.FatalIf(err, "Unable to parse Admin RPC Host")
 		rpcClient, err := NewAdminRPCClient(host)
-		logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
+		logger.FatalIf(err, "Unable to initialize Admin RPC Client")
 		adminPeerList = append(adminPeerList, adminPeer{
 			addr:      hostStr,
 			cmdRunner: rpcClient,

cmd/admin-rpc-server.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"context"
 	"path"
 
 	"github.com/gorilla/mux"
@@ -52,6 +51,23 @@ func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoDa
 	return err
 }
 
+// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
+type StartProfilingArgs struct {
+	AuthArgs
+	Profiler string
+}
+
+// StartProfiling - starts profiling of this server
+func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
+	return receiver.local.StartProfiling(args.Profiler)
+}
+
+// DownloadProfilingData - stops and returns profiling data of this server
+func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
+	*reply, err = receiver.local.DownloadProfilingData()
+	return
+}
+
 // GetConfig - returns the config.json of this server.
 func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
 	*reply, err = receiver.local.GetConfig()
@@ -81,7 +97,7 @@ func NewAdminRPCServer() (*xrpc.Server, error) {
 // registerAdminRPCRouter - creates and registers Admin RPC server and its router.
 func registerAdminRPCRouter(router *mux.Router) {
 	rpcServer, err := NewAdminRPCServer()
-	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(adminServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
 }

cmd/api-datatypes.go

@@ -69,7 +69,6 @@ type JSONType string
 // Constants for JSONTypes.
 const (
 	JSONDocumentType JSONType = "Document"
-	JSONStreamType            = "Stream"
 	JSONLinesType             = "Lines"
 )
 
@@ -80,6 +79,7 @@ type ObjectSelectRequest struct {
 	ExpressionType     QueryExpressionType
 	InputSerialization struct {
 		CompressionType SelectCompressionType
+		Parquet         *struct{}
 		CSV             *struct {
 			FileHeaderInfo       CSVFileHeaderInfo
 			RecordDelimiter      string
@@ -104,6 +104,9 @@ type ObjectSelectRequest struct {
 			RecordDelimiter string
 		}
 	}
+	RequestProgress struct {
+		Enabled bool
+	}
 }
 
 // ObjectIdentifier carries key name for the object to delete.

cmd/api-errors.go

@@ -291,6 +291,8 @@ const (
 	ErrInvalidColumnIndex
 	ErrMissingHeaders
 	ErrAdminConfigNotificationTargetsFailed
+	ErrAdminProfilerNotEnabled
+	ErrInvalidDecompressedSize
 )
 
 // error code to APIError structure, these fields carry respective
@@ -892,6 +894,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminProfilerNotEnabled: {
+		Code:           "XMinioAdminProfilerNotEnabled",
+		Description:    "Unable to perform the requested operation because profiling is not enabled",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminCredentialsMismatch: {
 		Code:           "XMinioAdminCredentialsMismatch",
 		Description:    "Credentials in config mismatch with server environment variables",
@@ -1397,6 +1404,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
 		Description:    "Some headers in the query are missing from the file. Check the file and try again.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidDecompressedSize: {
+		Code:           "XMinioInvalidDecompressedSize",
+		Description:    "The data provided is unfit for decompression",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	// Add your error structure here.
 }
 
@@ -1412,6 +1424,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 	switch err {
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
+	case errInvalidRange:
+		apiErr = ErrInvalidRange
 	case errDataTooLarge:
 		apiErr = ErrEntityTooLarge
 	case errDataTooSmall:
@@ -1616,6 +1630,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 	}
 
+	// Compression errors
+	switch err {
+	case errInvalidDecompressedSize:
+		apiErr = ErrInvalidDecompressedSize
+	}
+
 	if apiErr != ErrNone {
 		// If there was a match in the above switch case.
 		return apiErr
@@ -1727,7 +1747,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
 
 // getAPIError provides API Error for input API error code.
 func getAPIError(code APIErrorCode) APIError {
-	return errorCodeResponse[code]
+	if apiErr, ok := errorCodeResponse[code]; ok {
+		return apiErr
+	}
+	return errorCodeResponse[ErrInternalError]
 }
 
 // getErrorResponse gets in standard error and resource value and

cmd/api-headers.go

@@ -24,6 +24,8 @@ import (
 	"net/http"
 	"strconv"
 	"time"
+
+	"github.com/minio/minio/cmd/crypto"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -41,6 +43,9 @@ func setCommonHeaders(w http.ResponseWriter) {
 		w.Header().Set("X-Amz-Bucket-Region", region)
 	}
 	w.Header().Set("Accept-Ranges", "bytes")
+
+	// Remove sensitive information
+	crypto.RemoveSensitiveHeaders(w.Header())
 }
 
 // Encodes the response headers into XML format.
@@ -61,13 +66,10 @@ func encodeResponseJSON(response interface{}) []byte {
 }
 
 // Write object header
-func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
+func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec) (err error) {
 	// set common headers
 	setCommonHeaders(w)
 
-	// Set content length.
-	w.Header().Set("Content-Length", strconv.FormatInt(objInfo.Size, 10))
-
 	// Set last modified time.
 	lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat)
 	w.Header().Set("Last-Modified", lastModified)
@@ -95,10 +97,34 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
 		w.Header().Set(k, v)
 	}
 
-	// for providing ranged content
-	if contentRange != nil && contentRange.offsetBegin > -1 {
-		// Override content-length
-		w.Header().Set("Content-Length", strconv.FormatInt(contentRange.getLength(), 10))
-		w.Header().Set("Content-Range", contentRange.String())
+	var totalObjectSize int64
+	switch {
+	case crypto.IsEncrypted(objInfo.UserDefined):
+		totalObjectSize, err = objInfo.DecryptedSize()
+		if err != nil {
+			return err
+		}
+	case objInfo.IsCompressed():
+		totalObjectSize = objInfo.GetActualSize()
+		if totalObjectSize < 0 {
+			return errInvalidDecompressedSize
+		}
+	default:
+		totalObjectSize = objInfo.Size
 	}
+
+	// for providing ranged content
+	start, rangeLen, err := rs.GetOffsetLength(totalObjectSize)
+	if err != nil {
+		return err
+	}
+
+	// Set content length.
+	w.Header().Set("Content-Length", strconv.FormatInt(rangeLen, 10))
+	if rs != nil {
+		contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize)
+		w.Header().Set("Content-Range", contentRange)
+	}
+
+	return nil
 }

cmd/auth-handler.go

@@ -244,7 +244,7 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256))
+	reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
 		return toAPIErrorCode(err)
 	}

cmd/bitrot.go

@@ -138,7 +138,9 @@ func (b *bitrotReader) ReadChunk(offset int64, length int64) ([]byte, error) {
 	if b.buf == nil {
 		b.buf = make([]byte, b.endOffset-offset)
 		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(context.Background(), err)
+			ctx := context.Background()
+			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
+			logger.LogIf(ctx, err)
 			return nil, err
 		}
 	}

cmd/bucket-handlers-listobjects.go

@@ -102,7 +102,17 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
+		var actualSize int64
+		if listObjectsV2Info.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsV2Info.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsV2Info.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
 			listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -168,7 +178,17 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	}
 
 	for i := range listObjectsInfo.Objects {
-		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
+		var actualSize int64
+		if listObjectsInfo.Objects[i].IsCompressed() {
+			// Read the decompressed size from the meta.json.
+			actualSize = listObjectsInfo.Objects[i].GetActualSize()
+			if actualSize < 0 {
+				writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
+				return
+			}
+			// Set the info.Size to the actualSize.
+			listObjectsInfo.Objects[i].Size = actualSize
+		} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
 			listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -176,7 +196,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 			}
 		}
 	}
-
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, listObjectsInfo)
 
 	// Write success response.

cmd/bucket-handlers.go

@@ -590,7 +590,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
+	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(err), r.URL)
@@ -614,7 +614,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 			info := ObjectInfo{Size: fileSize}
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "") // do not try to verify encrypted content
+			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
 			if err != nil {
 				writeErrorResponse(w, toAPIErrorCode(err), r.URL)
 				return

cmd/bucket-handlers_test.go

@@ -84,7 +84,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize httptest Recorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get bucket location.
-		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -116,7 +116,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -220,7 +220,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		// construct HTTP request for HEAD bucket.
-		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, err := newTestSignedRequestV4("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -235,7 +235,7 @@ func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, api
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
-		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("HEAD", getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -437,7 +437,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 		// construct HTTP request for List multipart uploads endpoint.
 		u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads)
-		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		req, gerr := newTestSignedRequestV4("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if gerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", i+1, instanceType, gerr)
 		}
@@ -454,7 +454,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", u, 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -471,7 +471,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 
 	// construct HTTP request for List multipart uploads endpoint.
 	u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "")
-	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "") // Generate an anonymous request.
+	req, err := newTestSignedRequestV4("GET", u, 0, nil, "", "", nil) // Generate an anonymous request.
 	if err != nil {
 		t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: <ERROR> %v", instanceType, err)
 	}
@@ -551,7 +551,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 	for i, testCase := range testCases {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
-		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		req, lerr := newTestSignedRequestV4("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if lerr != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: <ERROR> %v", i+1, instanceType, lerr)
 		}
@@ -568,7 +568,7 @@ func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, ap
 		// construct HTTP request for PUT bucket policy endpoint.
 
 		// verify response for V2 signed HTTP request.
-		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey)
+		reqV2, err := newTestSignedRequestV2("GET", getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
@@ -745,7 +745,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		// Generate a signed or anonymous request based on the testCase
 		if testCase.accessKey != "" {
 			req, err = newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", bucketName),
-				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey)
+				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil)
 		} else {
 			req, err = newTestRequest("POST", getDeleteMultipleObjectsURL("", bucketName),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
@@ -785,7 +785,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	nilBucket := "dummy-bucket"
 	nilObject := ""
 
-	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "")
+	nilReq, err := newTestSignedRequestV4("POST", getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "", nil)
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
 	}

cmd/bucket-policy-handlers_test.go

@@ -254,7 +254,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -268,7 +268,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testCase.bucketName),
-			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey)
+			int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -304,7 +304,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -346,7 +346,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -360,7 +360,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -417,7 +417,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
@@ -456,7 +456,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("GET", getGetPolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -511,7 +511,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("GET", getGetPolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -591,7 +591,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -641,7 +641,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV4 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV4, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -663,7 +663,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("PUT", getPutPolicyURL("", testPolicy.bucketName),
-			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey)
+			int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -680,7 +680,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for Delete bucket policy endpoint.
 		reqV2, err := newTestSignedRequestV2("DELETE", getDeletePolicyURL("", testCase.bucketName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: <ERROR> %v", i+1, err)
 		}
@@ -714,7 +714,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 	nilBucket := "dummy-bucket"
 
 	nilReq, err := newTestSignedRequestV4("DELETE", getDeletePolicyURL("", nilBucket),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)

cmd/common-main.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	etcd "github.com/coreos/etcd/clientv3"
+	dns2 "github.com/miekg/dns"
 	"github.com/minio/cli"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
@@ -95,10 +96,23 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	setConfigDir(configDirAbs)
 }
 
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(includes []string) ([]string, error) {
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, uiErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+		}
+	}
+	return includes, nil
+}
+
 func handleCommonEnvVars() {
+	compressEnvDelimiter := ","
 	// Start profiler if env is set.
 	if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
-		globalProfiler = startProfiler(profiler)
+		var err error
+		globalProfiler, err = startProfiler(profiler, "")
+		logger.FatalIf(err, "Unable to setup a profiler")
 	}
 
 	accessKey := os.Getenv("MINIO_ACCESS_KEY")
@@ -117,7 +131,7 @@ func handleCommonEnvVars() {
 	if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
 		browserFlag, err := ParseBoolFlag(browser)
 		if err != nil {
-			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Unable to validate MINIO_BROWSER environment variable")
+			logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER environment variable")
 		}
 
 		// browser Envs are set globally, this does not represent
@@ -146,6 +160,11 @@ func handleCommonEnvVars() {
 	}
 
 	globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
+	if globalDomainName != "" {
+		if _, ok = dns2.IsDomainName(globalDomainName); !ok {
+			logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN environment variable")
+		}
+	}
 
 	minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
 	if ok {
@@ -260,4 +279,28 @@ func handleCommonEnvVars() {
 		globalKMSKeyID = kmsConf.Vault.Key.Name
 		globalKMSConfig = kmsConf
 	}
+
+	if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
+		globalIsCompressionEnabled = strings.EqualFold(compress, "true")
+	}
+
+	compressExtensions := os.Getenv("MINIO_COMPRESS_EXTENSIONS")
+	compressMimeTypes := os.Getenv("MINIO_COMPRESS_MIMETYPES")
+	if compressExtensions != "" || compressMimeTypes != "" {
+		globalIsEnvCompression = true
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", extensions)
+			}
+			globalCompressExtensions = extensions
+		}
+		if compressMimeTypes != "" {
+			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", contenttypes)
+			}
+			globalCompressMimeTypes = contenttypes
+		}
+	}
 }

cmd/config-current.go

@@ -23,7 +23,6 @@ import (
 	"reflect"
 	"sync"
 
-	"github.com/miekg/dns"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 
@@ -41,9 +40,9 @@ import (
 // 6. Make changes in config-current_test.go for any test change
 
 // Config version
-const serverConfigVersion = "28"
+const serverConfigVersion = "30"
 
-type serverConfig = serverConfigV28
+type serverConfig = serverConfigV30
 
 var (
 	// globalServerConfig server config.
@@ -97,12 +96,6 @@ func (s *serverConfig) GetCredential() auth.Credentials {
 	return s.Credential
 }
 
-// SetBrowser set if browser is enabled.
-func (s *serverConfig) SetBrowser(b bool) {
-	// Set the new value.
-	s.Browser = BoolFlag(b)
-}
-
 // SetWorm set if worm is enabled.
 func (s *serverConfig) SetWorm(b bool) {
 	// Set the new value.
@@ -126,17 +119,6 @@ func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
 	return s.StorageClass.Standard, s.StorageClass.RRS
 }
 
-// GetBrowser get current credentials.
-func (s *serverConfig) GetBrowser() bool {
-	if globalIsEnvBrowser {
-		return globalIsBrowserEnabled
-	}
-	if s == nil {
-		return true
-	}
-	return bool(s.Browser)
-}
-
 // GetWorm get current credentials.
 func (s *serverConfig) GetWorm() bool {
 	if globalIsEnvWORM {
@@ -188,14 +170,7 @@ func (s *serverConfig) Validate() error {
 	}
 
 	// Region: nothing to validate
-	// Browser, Worm, Cache and StorageClass values are already validated during json unmarshal
-
-	if s.Domain != "" {
-		if _, ok := dns.IsDomainName(s.Domain); !ok {
-			return errors.New("invalid domain name")
-		}
-	}
-
+	// Worm, Cache and StorageClass values are already validated during json unmarshal
 	for _, v := range s.Notify.AMQP {
 		if err := v.Validate(); err != nil {
 			return fmt.Errorf("amqp: %s", err.Error())
@@ -253,16 +228,24 @@ func (s *serverConfig) Validate() error {
 	return nil
 }
 
+// SetCompressionConfig sets the current compression config
+func (s *serverConfig) SetCompressionConfig(extensions []string, mimeTypes []string) {
+	s.Compression.Extensions = extensions
+	s.Compression.MimeTypes = mimeTypes
+	s.Compression.Enabled = globalIsCompressionEnabled
+}
+
+// GetCompressionConfig gets the current compression config
+func (s *serverConfig) GetCompressionConfig() compressionConfig {
+	return s.Compression
+}
+
 func (s *serverConfig) loadFromEnvs() {
 	// If env is set override the credentials from config file.
 	if globalIsEnvCreds {
 		s.SetCredential(globalActiveCred)
 	}
 
-	if globalIsEnvBrowser {
-		s.SetBrowser(globalIsBrowserEnabled)
-	}
-
 	if globalIsEnvWORM {
 		s.SetWorm(globalWORMEnabled)
 	}
@@ -271,10 +254,6 @@ func (s *serverConfig) loadFromEnvs() {
 		s.SetRegion(globalServerRegion)
 	}
 
-	if globalIsEnvDomainName {
-		s.Domain = globalDomainName
-	}
-
 	if globalIsStorageClass {
 		s.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
 	}
@@ -286,6 +265,10 @@ func (s *serverConfig) loadFromEnvs() {
 	if globalKMS != nil {
 		s.KMS = globalKMSConfig
 	}
+
+	if globalIsEnvCompression {
+		s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
+	}
 }
 
 // TestNotificationTargets tries to establish connections to all notification
@@ -395,14 +378,12 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
 		return "Credential configuration differs"
 	case s.Region != t.Region:
 		return "Region configuration differs"
-	case s.Browser != t.Browser:
-		return "Browser configuration differs"
-	case s.Domain != t.Domain:
-		return "Domain configuration differs"
 	case s.StorageClass != t.StorageClass:
 		return "StorageClass configuration differs"
 	case !reflect.DeepEqual(s.Cache, t.Cache):
 		return "Cache configuration differs"
+	case !reflect.DeepEqual(s.Compression, t.Compression):
+		return "Compression configuration differs"
 	case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
 		return "AMQP Notification configuration differs"
 	case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
@@ -442,7 +423,6 @@ func newServerConfig() *serverConfig {
 		Version:    serverConfigVersion,
 		Credential: cred,
 		Region:     globalMinioDefaultRegion,
-		Browser:    true,
 		StorageClass: storageClassConfig{
 			Standard: storageClass{},
 			RRS:      storageClass{},
@@ -455,6 +435,11 @@ func newServerConfig() *serverConfig {
 		},
 		KMS:    crypto.KMSConfig{},
 		Notify: notifier{},
+		Compression: compressionConfig{
+			Enabled:    false,
+			Extensions: globalCompressExtensions,
+			MimeTypes:  globalCompressMimeTypes,
+		},
 	}
 
 	// Make sure to initialize notification configs.
@@ -495,18 +480,12 @@ func (s *serverConfig) loadToCachedConfigs() {
 	if !globalIsEnvCreds {
 		globalActiveCred = s.GetCredential()
 	}
-	if !globalIsEnvBrowser {
-		globalIsBrowserEnabled = s.GetBrowser()
-	}
 	if !globalIsEnvWORM {
 		globalWORMEnabled = s.GetWorm()
 	}
 	if !globalIsEnvRegion {
 		globalServerRegion = s.GetRegion()
 	}
-	if !globalIsEnvDomainName {
-		globalDomainName = s.Domain
-	}
 	if !globalIsStorageClass {
 		globalStandardStorageClass, globalRRStorageClass = s.GetStorageClass()
 	}
@@ -524,6 +503,12 @@ func (s *serverConfig) loadToCachedConfigs() {
 			globalKMSKeyID = globalKMSConfig.Vault.Key.Name
 		}
 	}
+	if !globalIsCompressionEnabled {
+		compressionConf := s.GetCompressionConfig()
+		globalCompressExtensions = compressionConf.Extensions
+		globalCompressMimeTypes = compressionConf.MimeTypes
+		globalIsCompressionEnabled = compressionConf.Enabled
+	}
 }
 
 // newConfig - initialize a new server config, saves env parameters if

cmd/config-current_test.go

@@ -104,7 +104,7 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	initConfig(objLayer)
 
 	// Check if serverConfig has browser disabled
-	if globalServerConfig.GetBrowser() {
+	if globalIsBrowserEnabled {
 		t.Error("Expected browser to be disabled but it is not")
 	}
 
@@ -130,8 +130,8 @@ func TestServerConfigWithEnvs(t *testing.T) {
 	}
 
 	// Check if serverConfig has the correct domain
-	if globalServerConfig.Domain != "domain.com" {
-		t.Errorf("Expected Domain to be `domain.com`, found " + globalServerConfig.Domain)
+	if globalDomainName != "domain.com" {
+		t.Errorf("Expected Domain to be `domain.com`, found " + globalDomainName)
 	}
 }
 
@@ -267,70 +267,66 @@ func TestConfigDiff(t *testing.T) {
 		// 3
 		{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
 		// 4
-		{&serverConfig{Browser: false}, &serverConfig{Browser: true}, "Browser configuration differs"},
-		// 5
-		{&serverConfig{Domain: "domain1"}, &serverConfig{Domain: "domain2"}, "Domain configuration differs"},
-		// 6
 		{
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
 			&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
 			"StorageClass configuration differs",
 		},
-		// 7
+		// 5
 		{
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{AMQP: map[string]target.AMQPArgs{"1": {Enable: false}}}},
 			"AMQP Notification configuration differs",
 		},
-		// 8
+		// 6
 		{
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{NATS: map[string]target.NATSArgs{"1": {Enable: false}}}},
 			"NATS Notification configuration differs",
 		},
-		// 9
+		// 7
 		{
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
 			"ElasticSearch Notification configuration differs",
 		},
-		// 10
+		// 8
 		{
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
 			"Redis Notification configuration differs",
 		},
-		// 11
+		// 9
 		{
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
 			"PostgreSQL Notification configuration differs",
 		},
-		// 12
+		// 10
 		{
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
 			"Kafka Notification configuration differs",
 		},
-		// 13
+		// 11
 		{
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
 			"Webhook Notification configuration differs",
 		},
-		// 14
+		// 12
 		{
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
 			"MySQL Notification configuration differs",
 		},
-		// 15
+		// 13
 		{
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
 			&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
 			"MQTT Notification configuration differs",
 		},
-		// 16
+		// 14
 		{
 			&serverConfig{Logger: loggerConfig{
 				Console: loggerConsole{Enabled: true},

cmd/config-migrate.go

@@ -2387,6 +2387,7 @@ func migrateV27ToV28() error {
 
 	// config V28 is backward compatible with V27, load the old
 	// config file in serverConfigV28 struct and initialize KMSConfig
+
 	srvConfig := &serverConfigV28{}
 	_, err := quick.LoadConfig(configFile, globalEtcdClient, srvConfig)
 	if os.IsNotExist(err) {
@@ -2409,9 +2410,61 @@ func migrateV27ToV28() error {
 	return nil
 }
 
-// Migrates '.minio.sys/config.json' v27 to v28.
+// Migrates '.minio.sys/config.json' to v30.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
-	return migrateV27ToV28MinioSys(objAPI)
+	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
+		return err
+	}
+	if err := migrateV28ToV29MinioSys(objAPI); err != nil {
+		return err
+	}
+	return migrateV29ToV30MinioSys(objAPI)
+}
+
+func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+	srvConfig, err := readServerConfig(context.Background(), objAPI)
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if srvConfig.Version != "29" {
+		return nil
+	}
+
+	srvConfig.Version = "30"
+	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
+	srvConfig.Compression.Enabled = false
+	srvConfig.Compression.Extensions = globalCompressExtensions
+	srvConfig.Compression.MimeTypes = globalCompressMimeTypes
+	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from  29  to  30 . %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
+	return nil
+}
+
+func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+	srvConfig, err := readServerConfig(context.Background(), objAPI)
+	if err == errConfigNotFound {
+		return nil
+	} else if err != nil {
+		return fmt.Errorf("Unable to load config file. %v", err)
+	}
+	if srvConfig.Version != "28" {
+		return nil
+	}
+
+	srvConfig.Version = "29"
+	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
+		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
+	}
+
+	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
+	return nil
 }
 
 func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
@@ -2429,7 +2482,7 @@ func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	srvConfig.Version = "28"
 	srvConfig.KMS = crypto.KMSConfig{}
 	if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
+		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
 	}
 
 	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")

cmd/config-migrate_test.go

@@ -159,8 +159,8 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
 	}
 }
 
-// Test if a config migration from v2 to v28 is successfully done
-func TestServerConfigMigrateV2toV28(t *testing.T) {
+// Test if a config migration from v2 to v29 is successfully done
+func TestServerConfigMigrateV2toV29(t *testing.T) {
 	rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
 	if err != nil {
 		t.Fatal(err)
@@ -203,6 +203,10 @@ func TestServerConfigMigrateV2toV28(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
+	if err := migrateMinioSysConfig(objLayer); err != nil {
+		t.Fatal("Unexpected error: ", err)
+	}
+
 	// Initialize server config and check again if everything is fine
 	if err := loadConfig(objLayer); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
@@ -314,7 +318,6 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
 	if err := migrateV26ToV27(); err == nil {
 		t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
 	}
-
 	if err := migrateV27ToV28(); err == nil {
 		t.Fatal("migrateConfigV27ToV28() should fail with a corrupted json")
 	}

cmd/config-versions.go

@@ -737,9 +737,7 @@ type serverConfigV28 struct {
 	// S3 API configuration.
 	Credential auth.Credentials `json:"credential"`
 	Region     string           `json:"region"`
-	Browser    BoolFlag         `json:"browser"`
 	Worm       BoolFlag         `json:"worm"`
-	Domain     string           `json:"domain"`
 
 	// Storage class configuration
 	StorageClass storageClassConfig `json:"storageclass"`
@@ -756,3 +754,41 @@ type serverConfigV28 struct {
 	// Logger configuration
 	Logger loggerConfig `json:"logger"`
 }
+
+// compressionConfig represents the compression settings.
+type compressionConfig struct {
+	Enabled    bool     `json:"enabled"`
+	Extensions []string `json:"extensions"`
+	MimeTypes  []string `json:"mime-types"`
+}
+
+// serverConfigV30 is just like version '29', stores additionally
+// extensions and mimetypes fields for compression.
+type serverConfigV30 struct {
+	quick.Config `json:"-"` // ignore interfaces
+
+	Version string `json:"version"`
+
+	// S3 API configuration.
+	Credential auth.Credentials `json:"credential"`
+	Region     string           `json:"region"`
+	Worm       BoolFlag         `json:"worm"`
+
+	// Storage class configuration
+	StorageClass storageClassConfig `json:"storageclass"`
+
+	// Cache configuration
+	Cache CacheConfig `json:"cache"`
+
+	// KMS configuration
+	KMS crypto.KMSConfig `json:"kms"`
+
+	// Notification queue configuration.
+	Notify notifier `json:"notify"`
+
+	// Logger configuration
+	Logger loggerConfig `json:"logger"`
+
+	// Compression configuration
+	Compression compressionConfig `json:"compress"`
+}

cmd/config.go

@@ -160,8 +160,8 @@ func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
 	}
 
 	if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound, Quorum errors into errConfigNotFound
-		if isErrObjectNotFound(err) || isInsufficientReadQuorum(err) {
+		// Convert ObjectNotFound to errConfigNotFound
+		if isErrObjectNotFound(err) {
 			return errConfigNotFound
 		}
 		logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
@@ -172,7 +172,7 @@ func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
 }
 
 func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data))
+	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
 	if err != nil {
 		return err
 	}
@@ -187,8 +187,8 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*by
 	var buffer bytes.Buffer
 	// Read entire content by setting size to -1
 	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
-		// Convert ObjectNotFound, IncompleteBody and Quorum errors into errConfigNotFound
-		if isErrObjectNotFound(err) || isErrIncompleteBody(err) || isInsufficientReadQuorum(err) {
+		// Convert ObjectNotFound and IncompleteBody errors into errConfigNotFound
+		if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
 			return nil, errConfigNotFound
 		}
 
@@ -218,7 +218,32 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {
 		return errInvalidArgument
 	}
-	return initConfig(objAPI)
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing configuration needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	//  - Write quorum not met when upgrading configuration
+	//    version is needed.
+	retryTimerCh := newRetryTimerSimple(doneCh)
+	for {
+		select {
+		case _ = <-retryTimerCh:
+			err := initConfig(objAPI)
+			if err != nil {
+				if isInsufficientReadQuorum(err) || isInsufficientWriteQuorum(err) {
+					logger.Info("Waiting for configuration to be initialized..")
+					continue
+				}
+				return err
+			}
+
+			return nil
+		}
+	}
 }
 
 // NewConfigSys - creates new config system object.

cmd/copy-part-range.go

@@ -17,11 +17,8 @@
 package cmd
 
 import (
-	"fmt"
 	"net/http"
 	"net/url"
-	"strconv"
-	"strings"
 )
 
 // Writes S3 compatible copy part range error.
@@ -39,68 +36,33 @@ func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL) {
 	}
 }
 
-// Parses x-amz-copy-source-range for CopyObjectPart API. Specifically written to
-// differentiate the behavior between regular httpRange header v/s x-amz-copy-source-range.
-// The range of bytes to copy from the source object. The range value must use the form
-// bytes=first-last, where the first and last are the zero-based byte offsets to copy.
-// For example, bytes=0-9 indicates that you want to copy the first ten bytes of the source.
+// Parses x-amz-copy-source-range for CopyObjectPart API. Its behavior
+// is different from regular HTTP range header. It only supports the
+// form `bytes=first-last` where first and last are zero-based byte
+// offsets. See
 // http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html
-func parseCopyPartRange(rangeString string, resourceSize int64) (hrange *httpRange, err error) {
-	// Return error if given range string doesn't start with byte range prefix.
-	if !strings.HasPrefix(rangeString, byteRangePrefix) {
-		return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix)
+// for full details. This function treats an empty rangeString as
+// referring to the whole resource.
+func parseCopyPartRangeSpec(rangeString string) (hrange *HTTPRangeSpec, err error) {
+	hrange, err = parseRequestRangeSpec(rangeString)
+	if err != nil {
+		return nil, err
 	}
-
-	// Trim byte range prefix.
-	byteRangeString := strings.TrimPrefix(rangeString, byteRangePrefix)
-
-	// Check if range string contains delimiter '-', else return error. eg. "bytes=8"
-	sepIndex := strings.Index(byteRangeString, "-")
-	if sepIndex == -1 {
+	if hrange.IsSuffixLength || hrange.Start < 0 || hrange.End < 0 {
 		return nil, errInvalidRange
 	}
-
-	offsetBeginString := byteRangeString[:sepIndex]
-	offsetBegin := int64(-1)
-	// Convert offsetBeginString only if its not empty.
-	if len(offsetBeginString) > 0 {
-		if !validBytePos.MatchString(offsetBeginString) {
-			return nil, errInvalidRange
-		}
-		if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil {
-			return nil, errInvalidRange
-		}
-	}
-
-	offsetEndString := byteRangeString[sepIndex+1:]
-	offsetEnd := int64(-1)
-	// Convert offsetEndString only if its not empty.
-	if len(offsetEndString) > 0 {
-		if !validBytePos.MatchString(offsetEndString) {
-			return nil, errInvalidRange
-		}
-		if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil {
-			return nil, errInvalidRange
-		}
-	}
-
-	// rangeString contains first byte positions. eg. "bytes=2-" or
-	// rangeString contains last bye positions. eg. "bytes=-2"
-	if offsetBegin == -1 || offsetEnd == -1 {
-		return nil, errInvalidRange
-	}
-
-	// Last byte position should not be greater than first byte
-	// position. eg. "bytes=5-2"
-	if offsetBegin > offsetEnd {
-		return nil, errInvalidRange
-	}
-
-	// First and last byte positions should not be >= resourceSize.
-	if offsetBegin >= resourceSize || offsetEnd >= resourceSize {
-		return nil, errInvalidRangeSource
-	}
-
-	// Success..
-	return &httpRange{offsetBegin, offsetEnd, resourceSize}, nil
+	return hrange, nil
+}
+
+// checkCopyPartRangeWithSize adds more check to the range string in case of
+// copy object part. This API requires having specific start and end  range values
+// e.g. 'bytes=3-10'. Other use cases will be rejected.
+func checkCopyPartRangeWithSize(rs *HTTPRangeSpec, resourceSize int64) (err error) {
+	if rs == nil {
+		return nil
+	}
+	if rs.IsSuffixLength || rs.Start >= resourceSize || rs.End >= resourceSize {
+		return errInvalidRangeSource
+	}
+	return nil
 }

cmd/copy-part-range_test.go

@@ -19,35 +19,37 @@ package cmd
 import "testing"
 
 // Test parseCopyPartRange()
-func TestParseCopyPartRange(t *testing.T) {
+func TestParseCopyPartRangeSpec(t *testing.T) {
 	// Test success cases.
 	successCases := []struct {
 		rangeString string
 		offsetBegin int64
 		offsetEnd   int64
-		length      int64
 	}{
-		{"bytes=2-5", 2, 5, 4},
-		{"bytes=2-9", 2, 9, 8},
-		{"bytes=2-2", 2, 2, 1},
-		{"bytes=0000-0006", 0, 6, 7},
+		{"bytes=2-5", 2, 5},
+		{"bytes=2-9", 2, 9},
+		{"bytes=2-2", 2, 2},
+		{"bytes=0000-0006", 0, 6},
 	}
+	objectSize := int64(10)
 
 	for _, successCase := range successCases {
-		hrange, err := parseCopyPartRange(successCase.rangeString, 10)
+		rs, err := parseCopyPartRangeSpec(successCase.rangeString)
 		if err != nil {
 			t.Fatalf("expected: <nil>, got: %s", err)
 		}
 
-		if hrange.offsetBegin != successCase.offsetBegin {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, hrange.offsetBegin)
+		start, length, err1 := rs.GetOffsetLength(objectSize)
+		if err1 != nil {
+			t.Fatalf("expected: <nil>, got: %s", err1)
 		}
 
-		if hrange.offsetEnd != successCase.offsetEnd {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, hrange.offsetEnd)
+		if start != successCase.offsetBegin {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, start)
 		}
-		if hrange.getLength() != successCase.length {
-			t.Fatalf("expected: %d, got: %d", successCase.length, hrange.getLength())
+
+		if start+length-1 != successCase.offsetEnd {
+			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, start+length-1)
 		}
 	}
 
@@ -59,15 +61,16 @@ func TestParseCopyPartRange(t *testing.T) {
 		"bytes=2-+5",
 		"bytes=2--5",
 		"bytes=-",
-		"",
 		"2-5",
 		"bytes = 2-5",
 		"bytes=2 - 5",
 		"bytes=0-0,-1",
 		"bytes=2-5 ",
+		"bytes=-1",
+		"bytes=1-",
 	}
 	for _, rangeString := range invalidRangeStrings {
-		if _, err := parseCopyPartRange(rangeString, 10); err == nil {
+		if _, err := parseCopyPartRangeSpec(rangeString); err == nil {
 			t.Fatalf("expected: an error, got: <nil> for range %s", rangeString)
 		}
 	}
@@ -78,8 +81,14 @@ func TestParseCopyPartRange(t *testing.T) {
 		"bytes=20-30",
 	}
 	for _, rangeString := range errorRangeString {
-		if _, err := parseCopyPartRange(rangeString, 10); err != errInvalidRangeSource {
-			t.Fatalf("expected: %s, got: %s", errInvalidRangeSource, err)
+		rs, err := parseCopyPartRangeSpec(rangeString)
+		if err == nil {
+			err1 := checkCopyPartRangeWithSize(rs, objectSize)
+			if err1 != errInvalidRangeSource {
+				t.Fatalf("expected: %s, got: %s", errInvalidRangeSource, err)
+			}
+		} else {
+			t.Fatalf("expected: %s, got: <nil>", errInvalidRangeSource)
 		}
 	}
 }

cmd/crypto/header.go

@@ -74,6 +74,14 @@ const (
 	SSEAlgorithmKMS = "aws:kms"
 )
 
+// RemoveSensitiveHeaders removes confidential encryption
+// information - e.g. the SSE-C key - from the HTTP headers.
+// It has the same semantics as RemoveSensitiveEntires.
+func RemoveSensitiveHeaders(h http.Header) {
+	h.Del(SSECKey)
+	h.Del(SSECopyKey)
+}
+
 // S3 represents AWS SSE-S3. It provides functionality to handle
 // SSE-S3 requests.
 var S3 = s3{}
@@ -164,7 +172,6 @@ func (ssecCopy) IsRequested(h http.Header) bool {
 // ParseHTTP parses the SSE-C headers and returns the SSE-C client key
 // on success. SSE-C copy headers are ignored.
 func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	defer h.Del(SSECKey) // remove SSE-C key from headers after parsing
 	if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
 		return key, ErrInvalidCustomerAlgorithm
 	}
@@ -190,7 +197,6 @@ func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
 // ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
 // on success. Regular SSE-C headers are ignored.
 func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
-	defer h.Del(SSECopyKey) // remove SSE-C copy key of source object from headers after parsing
 	if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
 		return key, ErrInvalidCustomerAlgorithm
 	}

cmd/crypto/header_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"net/http"
+	"sort"
 	"testing"
 )
 
@@ -250,9 +251,6 @@ func TestSSECParse(t *testing.T) {
 		if err == nil && key == zeroKey {
 			t.Errorf("Test %d: parsed client key is zero key", i)
 		}
-		if _, ok := test.Header[SSECKey]; ok {
-			t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
-		}
 	}
 }
 
@@ -337,3 +335,102 @@ func TestSSECopyParse(t *testing.T) {
 		}
 	}
 }
+
+var removeSensitiveHeadersTests = []struct {
+	Header, ExpectedHeader http.Header
+}{
+	{
+		Header: http.Header{
+			SSECKey:    []string{""},
+			SSECopyKey: []string{""},
+		},
+		ExpectedHeader: http.Header{},
+	},
+	{ // Standard SSE-C request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + SSE-C-copy request headers
+		Header: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKey:       []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKey:    []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm: []string{SSEAlgorithmAES256},
+			SSECKeyMD5:    []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
+		},
+	},
+	{ // Standard SSE-C + metadata request headers
+		Header: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKey:             []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+		ExpectedHeader: http.Header{
+			SSECAlgorithm:       []string{SSEAlgorithmAES256},
+			SSECKeyMD5:          []string{"7PpPLAK26ONlVUGOWlusfg=="},
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+	},
+}
+
+func TestRemoveSensitiveHeaders(t *testing.T) {
+	isEqual := func(x, y http.Header) bool {
+		if len(x) != len(y) {
+			return false
+		}
+		for k, v := range x {
+			u, ok := y[k]
+			if !ok || len(v) != len(u) {
+				return false
+			}
+			sort.Strings(v)
+			sort.Strings(u)
+			for j := range v {
+				if v[j] != u[j] {
+					return false
+				}
+			}
+		}
+		return true
+	}
+	areKeysEqual := func(h http.Header, metadata map[string]string) bool {
+		if len(h) != len(metadata) {
+			return false
+		}
+		for k := range h {
+			if _, ok := metadata[k]; !ok {
+				return false
+			}
+		}
+		return true
+	}
+
+	for i, test := range removeSensitiveHeadersTests {
+		metadata := make(map[string]string, len(test.Header))
+		for k := range test.Header {
+			metadata[k] = "" // set metadata key - we don't care about the value
+		}
+
+		RemoveSensitiveHeaders(test.Header)
+		if !isEqual(test.ExpectedHeader, test.Header) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+		RemoveSensitiveEntries(metadata)
+		if !areKeysEqual(test.ExpectedHeader, metadata) {
+			t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader)
+		}
+	}
+}

cmd/crypto/metadata.go

@@ -32,6 +32,14 @@ func IsMultiPart(metadata map[string]string) bool {
 	return false
 }
 
+// RemoveSensitiveEntries removes confidential encryption
+// information - e.g. the SSE-C key - from the metadata map.
+// It has the same semantics as RemoveSensitiveHeaders.
+func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
+	delete(metadata, SSECKey)
+	delete(metadata, SSECopyKey)
+}
+
 // IsEncrypted returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption.
 //

cmd/crypto/metadata_test.go

@@ -326,15 +326,20 @@ func TestS3CreateMetadata(t *testing.T) {
 	_ = S3.CreateMetadata(nil, "", []byte{}, SealedKey{Algorithm: InsecureSealAlgorithm})
 }
 
-var ssecCreateMetadataTests = []SealedKey{
-	{Algorithm: SealAlgorithm},
-	{IV: [32]byte{0xff}, Key: [64]byte{0x7e}, Algorithm: SealAlgorithm},
+var ssecCreateMetadataTests = []struct {
+	KeyID         string
+	SealedDataKey []byte
+	SealedKey     SealedKey
+}{
+	{KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}},
+	{KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}},
 }
 
 func TestSSECCreateMetadata(t *testing.T) {
 	defer func(disableLog bool) { logger.Disable = disableLog }(logger.Disable)
 	logger.Disable = true
-	for i, test := range s3CreateMetadataTests {
+	for i, test := range ssecCreateMetadataTests {
 		metadata := SSEC.CreateMetadata(nil, test.SealedKey)
 		sealedKey, err := SSEC.ParseMetadata(metadata)
 		if err != nil {

cmd/crypto/vault.go

@@ -28,8 +28,8 @@ import (
 )
 
 const (
-	// VaultEndpointEnv Vault endpoint environment variable
-	VaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
+	// vaultEndpointEnv Vault endpoint environment variable
+	vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
 	// vaultAuthTypeEnv type of vault auth to be used
 	vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
 	// vaultAppRoleIDEnv  Vault AppRole ID environment variable
@@ -40,6 +40,10 @@ const (
 	vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
 	// vaultKeyNameEnv Vault Encryption Key Name environment variable
 	vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
+
+	// vaultCAPath is the path to a directory of PEM-encoded CA
+	// cert files to verify the Vault server SSL certificate.
+	vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
 )
 
 var (
@@ -93,7 +97,7 @@ type VaultConfig struct {
 // been set
 func validateVaultConfig(c *VaultConfig) error {
 	if c.Endpoint == "" {
-		return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", VaultEndpointEnv)
+		return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
 	}
 	if strings.ToLower(c.Auth.Type) != "approle" {
 		return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
@@ -110,7 +114,6 @@ func validateVaultConfig(c *VaultConfig) error {
 	if c.Key.Version < 0 {
 		return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
 	}
-
 	return nil
 }
 
@@ -134,7 +137,7 @@ func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (tok
 // variables and performs validations.
 func NewVaultConfig() (KMSConfig, error) {
 	kc := KMSConfig{}
-	endpoint := os.Getenv(VaultEndpointEnv)
+	endpoint := os.Getenv(vaultEndpointEnv)
 	roleID := os.Getenv(vaultAppRoleIDEnv)
 	roleSecret := os.Getenv(vaultAppSecretIDEnv)
 	keyName := os.Getenv(vaultKeyNameEnv)
@@ -177,9 +180,15 @@ func NewVaultConfig() (KMSConfig, error) {
 // and gets a client token for future api calls.
 func NewVault(kmsConf KMSConfig) (KMS, error) {
 	config := kmsConf.Vault
-	c, err := vault.NewClient(&vault.Config{
+	vconfig := &vault.Config{
 		Address: config.Endpoint,
-	})
+	}
+	if err := vconfig.ConfigureTLS(&vault.TLSConfig{
+		CAPath: os.Getenv(vaultCAPath),
+	}); err != nil {
+		return nil, err
+	}
+	c, err := vault.NewClient(vconfig)
 	if err != nil {
 		return nil, err
 	}

cmd/disk-cache-fs.go

@@ -354,7 +354,7 @@ func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object
 	}
 
 	// Validate input data size and it can never be less than zero.
-	if data.Size() < 0 {
+	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return ObjectInfo{}, errInvalidArgument
 	}

cmd/disk-cache.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/djherbis/atime"
 
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/wildcard"
@@ -57,6 +58,7 @@ type cacheObjects struct {
 	// file path patterns to exclude from cache
 	exclude []string
 	// Object functions pointing to the corresponding functions of backend implementation.
+	GetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectFn               func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	PutObjectFn               func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -88,6 +90,7 @@ type CacheObjectLayer interface {
 	ListBuckets(ctx context.Context) (buckets []BucketInfo, err error)
 	DeleteBucket(ctx context.Context, bucket string) error
 	// Object operations.
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -103,6 +106,11 @@ type CacheObjectLayer interface {
 	StorageInfo(ctx context.Context) CacheStorageInfo
 }
 
+// IsCacheable returns if the object should be saved in the cache.
+func (o ObjectInfo) IsCacheable() bool {
+	return !crypto.IsEncrypted(o.UserDefined)
+}
+
 // backendDownError returns true if err is due to backend failure or faulty disk if in server mode
 func backendDownError(err error) bool {
 	_, backendDown := err.(BackendDown)
@@ -175,6 +183,88 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string {
 	return metadata
 }
 
+func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+
+	objInfo, bkErr := c.GetObjectInfoFn(ctx, bucket, object, opts)
+
+	if c.isCacheExclude(bucket, object) || !objInfo.IsCacheable() {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	// fetch cacheFSObjects if object is currently cached or nearest available cache drive
+	dcache, err := c.cache.getCachedFSLoc(ctx, bucket, object)
+	if err != nil {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	backendDown := backendDownError(bkErr)
+	if bkErr != nil && !backendDown {
+		if _, ok := err.(ObjectNotFound); ok {
+			// Delete the cached entry if backend object was deleted.
+			dcache.Delete(ctx, bucket, object)
+		}
+		return nil, bkErr
+	}
+
+	if !backendDown && filterFromCache(objInfo.UserDefined) {
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	if cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts); cacheErr == nil {
+		if backendDown {
+			// If the backend is down, serve the request from cache.
+			return cacheReader, nil
+		}
+
+		if cacheReader.ObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
+			// Object is not stale, so serve from cache
+			return cacheReader, nil
+		}
+
+		// Object is stale, so delete from cache
+		dcache.Delete(ctx, bucket, object)
+	}
+
+	// Since we got here, we are serving the request from backend,
+	// and also adding the object to the cache.
+
+	if rs != nil {
+		// We don't cache partial objects.
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+	if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
+		// cache only objects < 1/100th of disk capacity
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	}
+
+	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
+	if bkErr != nil {
+		return nil, bkErr
+	}
+
+	// Initialize pipe.
+	pipeReader, pipeWriter := io.Pipe()
+	teeReader := io.TeeReader(bkReader, pipeWriter)
+	hashReader, herr := hash.NewReader(pipeReader, bkReader.ObjInfo.Size, "", "", bkReader.ObjInfo.Size)
+	if herr != nil {
+		bkReader.Close()
+		return nil, herr
+	}
+
+	go func() {
+		opts := ObjectOptions{}
+		putErr := dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(bkReader.ObjInfo), opts)
+		// close the write end of the pipe, so the error gets
+		// propagated to getObjReader
+		pipeWriter.CloseWithError(putErr)
+	}()
+
+	cleanupBackend := func() { bkReader.Close() }
+	cleanupPipe := func() { pipeReader.Close() }
+	gr = NewGetObjectReaderFromReader(teeReader, bkReader.ObjInfo, cleanupBackend, cleanupPipe)
+	return gr, nil
+}
+
 // Uses cached-object to serve the request. If object is not cached it serves the request from the backend and also
 // stores it in the cache for serving subsequent requests.
 func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
@@ -225,16 +315,13 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
 	}
 	// Initialize pipe.
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "")
+	hashReader, err := hash.NewReader(pipeReader, objInfo.Size, "", "", objInfo.Size)
 	if err != nil {
 		return err
 	}
 	go func() {
-		if err = GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts); err != nil {
-			pipeWriter.CloseWithError(err)
-			return
-		}
-		pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
+		gerr := GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts)
+		pipeWriter.CloseWithError(gerr) // Close writer explicitly signaling we wrote all data.
 	}()
 	err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo), opts)
 	if err != nil {
@@ -582,13 +669,13 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
 	objInfo = ObjectInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, r.MD5HexString(), r.SHA256HexString())
+	hashReader, err := hash.NewReader(pipeReader, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, r.MD5HexString(), r.SHA256HexString())
+	cHashReader, err := hash.NewReader(rPipe, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -675,13 +762,13 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
 	info = PartInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString())
+	hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return
 	}
 	// Initialize pipe to stream data to cache
 	rPipe, wPipe := io.Pipe()
-	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString())
+	cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
 	if err != nil {
 		return
 	}

cmd/disk-cache_test.go

@@ -195,7 +195,7 @@ func TestDiskCache(t *testing.T) {
 	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
-	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -270,7 +270,7 @@ func TestDiskCacheMaxUse(t *testing.T) {
 	opts := ObjectOptions{}
 
 	byteReader := bytes.NewReader([]byte(content))
-	hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
+	hashReader, err := hash.NewReader(byteReader, int64(size), "", "", int64(size))
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/dummy-data-generator_test.go

@@ -0,0 +1,182 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+var alphabets = []byte("abcdefghijklmnopqrstuvwxyz0123456789")
+
+// DummyDataGen returns a reader that repeats the bytes in `alphabets`
+// upto the desired length.
+type DummyDataGen struct {
+	b           []byte
+	idx, length int64
+}
+
+// NewDummyDataGen returns a ReadSeeker over the first `totalLength`
+// bytes from the infinite stream consisting of repeated
+// concatenations of `alphabets`.
+//
+// The skipOffset (generally = 0) can be used to skip a given number
+// of bytes from the beginning of the infinite stream. This is useful
+// to compare such streams of bytes that may be split up, because:
+//
+// Given the function:
+//
+// f := func(r io.Reader) string {
+//           b, _ := ioutil.ReadAll(r)
+//           return string(b)
+// }
+//
+// for example, the following is true:
+//
+// f(NewDummyDataGen(100, 0)) == f(NewDummyDataGen(50, 0)) + f(NewDummyDataGen(50, 50))
+func NewDummyDataGen(totalLength, skipOffset int64) io.ReadSeeker {
+	if totalLength < 0 {
+		panic("Negative length passed to DummyDataGen!")
+	}
+	if skipOffset < 0 {
+		panic("Negative rotations are not allowed")
+	}
+
+	skipOffset = skipOffset % int64(len(alphabets))
+	as := make([]byte, 2*len(alphabets))
+	copy(as, alphabets)
+	copy(as[len(alphabets):], alphabets)
+	b := as[skipOffset : skipOffset+int64(len(alphabets))]
+	return &DummyDataGen{
+		length: totalLength,
+		b:      b,
+	}
+}
+
+func (d *DummyDataGen) Read(b []byte) (n int, err error) {
+	k := len(b)
+	numLetters := int64(len(d.b))
+	for k > 0 && d.idx < d.length {
+		w := copy(b[len(b)-k:], d.b[d.idx%numLetters:])
+		k -= w
+		d.idx += int64(w)
+		n += w
+	}
+	if d.idx >= d.length {
+		extraBytes := d.idx - d.length
+		n -= int(extraBytes)
+		if n < 0 {
+			n = 0
+		}
+		err = io.EOF
+	}
+	return
+}
+
+func (d *DummyDataGen) Seek(offset int64, whence int) (int64, error) {
+	switch whence {
+	case io.SeekStart:
+		if offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx = offset
+	case io.SeekCurrent:
+		if d.idx+offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx += offset
+	case io.SeekEnd:
+		if d.length+offset < 0 {
+			return 0, errors.New("Invalid offset")
+		}
+		d.idx = d.length + offset
+	}
+	return d.idx, nil
+}
+
+func TestDummyDataGenerator(t *testing.T) {
+	readAll := func(r io.Reader) string {
+		b, _ := ioutil.ReadAll(r)
+		return string(b)
+	}
+	checkEq := func(a, b string) {
+		if a != b {
+			t.Fatalf("Unexpected equality failure")
+		}
+	}
+
+	checkEq(readAll(NewDummyDataGen(0, 0)), "")
+
+	checkEq(readAll(NewDummyDataGen(10, 0)), readAll(NewDummyDataGen(10, int64(len(alphabets)))))
+
+	checkEq(readAll(NewDummyDataGen(100, 0)), readAll(NewDummyDataGen(50, 0))+readAll(NewDummyDataGen(50, 50)))
+
+	r := NewDummyDataGen(100, 0)
+	r.Seek(int64(len(alphabets)), 0)
+	checkEq(readAll(r), readAll(NewDummyDataGen(100-int64(len(alphabets)), 0)))
+}
+
+// Compares all the bytes returned by the given readers. Any Read
+// errors cause a `false` result. A string describing the error is
+// also returned.
+func cmpReaders(r1, r2 io.Reader) (bool, string) {
+	bufLen := 32 * 1024
+	b1, b2 := make([]byte, bufLen), make([]byte, bufLen)
+	for i := 0; true; i++ {
+		n1, e1 := io.ReadFull(r1, b1)
+		n2, e2 := io.ReadFull(r2, b2)
+		if n1 != n2 {
+			return false, fmt.Sprintf("Read %d != %d bytes from the readers", n1, n2)
+		}
+		if !bytes.Equal(b1[:n1], b2[:n2]) {
+			return false, fmt.Sprintf("After reading %d equal buffers (32Kib each), we got the following two strings:\n%v\n%v\n",
+				i, b1, b2)
+		}
+		// Check if stream has ended
+		if (e1 == io.ErrUnexpectedEOF && e2 == io.ErrUnexpectedEOF) || (e1 == io.EOF && e2 == io.EOF) {
+			break
+		}
+		if e1 != nil || e2 != nil {
+			return false, fmt.Sprintf("Got unexpected error values: %v == %v", e1, e2)
+		}
+	}
+	return true, ""
+}
+
+func TestCmpReaders(t *testing.T) {
+	{
+		r1 := bytes.NewBuffer([]byte("abc"))
+		r2 := bytes.NewBuffer([]byte("abc"))
+		ok, msg := cmpReaders(r1, r2)
+		if !(ok && msg == "") {
+			t.Fatalf("unexpected")
+		}
+	}
+
+	{
+		r1 := bytes.NewBuffer([]byte("abc"))
+		r2 := bytes.NewBuffer([]byte("abcd"))
+		ok, _ := cmpReaders(r1, r2)
+		if ok {
+			t.Fatalf("unexpected")
+		}
+	}
+}

cmd/dummy-object-layer_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"io"
+	"net/http"
 
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/madmin"
@@ -59,6 +60,10 @@ func (api *DummyObjectLayer) ListObjectsV2(ctx context.Context, bucket, prefix,
 	return
 }
 
+func (api *DummyObjectLayer) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lock LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	return
+}
+
 func (api *DummyObjectLayer) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) {
 	return
 }
@@ -154,3 +159,7 @@ func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
 func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
 	return
 }
+
+func (api *DummyObjectLayer) IsCompressionSupported() (b bool) {
+	return
+}

cmd/encryption-v1.go

@@ -82,7 +82,7 @@ func hasServerSideEncryptionHeader(header http.Header) bool {
 
 // ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
 // It returns the client provided key on success.
-func ParseSSECopyCustomerRequest(r *http.Request, metadata map[string]string) (key []byte, err error) {
+func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {
 	if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
 		// we cannot use r.TLS == nil here because Go's http implementation reflects on
 		// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
@@ -90,10 +90,10 @@ func ParseSSECopyCustomerRequest(r *http.Request, metadata map[string]string) (k
 		// will always fail -> r.TLS is always nil even for TLS requests.
 		return nil, errInsecureSSERequest
 	}
-	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(r.Header) {
+	if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {
 		return nil, crypto.ErrIncompatibleEncryptionMethod
 	}
-	k, err := crypto.SSECopy.ParseHTTP(r.Header)
+	k, err := crypto.SSECopy.ParseHTTP(h)
 	return k[:], err
 }
 
@@ -123,8 +123,6 @@ func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
 
 // This function rotates old to new key.
 func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map[string]string) error {
-	delete(metadata, crypto.SSECKey) // make sure we do not save the key by accident
-
 	switch {
 	default:
 		return errObjectTampered
@@ -155,8 +153,6 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
 }
 
 func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) ([]byte, error) {
-	delete(metadata, crypto.SSECKey) // make sure we do not save the key by accident
-
 	var sealedKey crypto.SealedKey
 	if sseS3 {
 		if globalKMS == nil {
@@ -240,12 +236,11 @@ func DecryptCopyRequest(client io.Writer, r *http.Request, bucket, object string
 		err error
 	)
 	if crypto.SSECopy.IsRequested(r.Header) {
-		key, err = ParseSSECopyCustomerRequest(r, metadata)
+		key, err = ParseSSECopyCustomerRequest(r.Header, metadata)
 		if err != nil {
 			return nil, err
 		}
 	}
-	delete(metadata, crypto.SSECopyKey) // make sure we do not save the key by accident
 	return newDecryptWriter(client, key, bucket, object, 0, metadata)
 }
 
@@ -312,6 +307,125 @@ func newDecryptWriterWithObjectKey(client io.Writer, objectEncryptionKey []byte,
 	return writer, nil
 }
 
+// Adding support for reader based interface
+
+// DecryptRequestWithSequenceNumberR - same as
+// DecryptRequestWithSequenceNumber but with a reader
+func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	if crypto.S3.IsEncrypted(metadata) {
+		return newDecryptReader(client, nil, bucket, object, seqNumber, metadata)
+	}
+
+	key, err := ParseSSECustomerHeader(h)
+	if err != nil {
+		return nil, err
+	}
+	return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
+}
+
+// DecryptCopyRequestR - same as DecryptCopyRequest, but with a
+// Reader
+func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, metadata map[string]string) (io.Reader, error) {
+	var (
+		key []byte
+		err error
+	)
+	if crypto.SSECopy.IsRequested(h) {
+		key, err = ParseSSECopyCustomerRequest(h, metadata)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return newDecryptReader(client, key, bucket, object, 0, metadata)
+}
+
+func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	objectEncryptionKey, err := decryptObjectInfo(key, bucket, object, metadata)
+	if err != nil {
+		return nil, err
+	}
+	return newDecryptReaderWithObjectKey(client, objectEncryptionKey, seqNumber, metadata)
+}
+
+func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
+	reader, err := sio.DecryptReader(client, sio.Config{
+		Key:            objectEncryptionKey,
+		SequenceNumber: seqNumber,
+	})
+	if err != nil {
+		return nil, crypto.ErrInvalidCustomerKey
+	}
+	return reader, nil
+}
+
+// GetEncryptedOffsetLength - returns encrypted offset and length
+// along with sequence number
+func GetEncryptedOffsetLength(startOffset, length int64, objInfo ObjectInfo) (seqNumber uint32, encStartOffset, encLength int64) {
+	if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
+		seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
+		return
+	}
+	seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
+	return
+}
+
+// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
+// reader
+func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
+	length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
+	io.Reader, error) {
+
+	bucket, object := oi.Bucket, oi.Name
+
+	// Single part case
+	if len(oi.Parts) == 0 || !crypto.IsMultiPart(oi.UserDefined) {
+		var reader io.Reader
+		var err error
+		if copySource {
+			reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, oi.UserDefined)
+		} else {
+			reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
+		}
+		if err != nil {
+			return nil, err
+		}
+		return reader, nil
+	}
+
+	partDecRelOffset := int64(seqNumber) * sseDAREPackageBlockSize
+	partEncRelOffset := int64(seqNumber) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+
+	w := &DecryptBlocksReader{
+		reader:            inputReader,
+		startSeqNum:       seqNumber,
+		partDecRelOffset:  partDecRelOffset,
+		partEncRelOffset:  partEncRelOffset,
+		parts:             oi.Parts,
+		partIndex:         partStart,
+		header:            h,
+		bucket:            bucket,
+		object:            object,
+		customerKeyHeader: h.Get(crypto.SSECKey),
+		copySource:        copySource,
+	}
+
+	w.metadata = map[string]string{}
+	// Copy encryption metadata for internal use.
+	for k, v := range oi.UserDefined {
+		w.metadata[k] = v
+	}
+
+	if w.copySource {
+		w.customerKeyHeader = h.Get(crypto.SSECopyKey)
+	}
+
+	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
+		return nil, err
+	}
+
+	return w, nil
+}
+
 // DecryptRequestWithSequenceNumber decrypts the object with the client provided key. It also removes
 // the client-side-encryption metadata from the object and sets the correct headers.
 func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket, object string, seqNumber uint32, metadata map[string]string) (io.WriteCloser, error) {
@@ -323,7 +437,6 @@ func DecryptRequestWithSequenceNumber(client io.Writer, r *http.Request, bucket,
 	if err != nil {
 		return nil, err
 	}
-	delete(metadata, crypto.SSECKey) // make sure we do not save the key by accident
 	return newDecryptWriter(client, key, bucket, object, seqNumber, metadata)
 }
 
@@ -333,7 +446,124 @@ func DecryptRequest(client io.Writer, r *http.Request, bucket, object string, me
 	return DecryptRequestWithSequenceNumber(client, r, bucket, object, 0, metadata)
 }
 
-// DecryptBlocksWriter - decrypts multipart parts, while implementing a io.Writer compatible interface.
+// DecryptBlocksReader - decrypts multipart parts, while implementing
+// a io.Reader compatible interface.
+type DecryptBlocksReader struct {
+	// Source of the encrypted content that will be decrypted
+	reader io.Reader
+	// Current decrypter for the current encrypted data block
+	decrypter io.Reader
+	// Start sequence number
+	startSeqNum uint32
+	// Current part index
+	partIndex int
+	// Parts information
+	parts          []objectPartInfo
+	header         http.Header
+	bucket, object string
+	metadata       map[string]string
+
+	partDecRelOffset, partEncRelOffset int64
+
+	copySource bool
+	// Customer Key
+	customerKeyHeader string
+}
+
+func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
+	m := make(map[string]string)
+	for k, v := range d.metadata {
+		m[k] = v
+	}
+	// Initialize the first decrypter; new decrypters will be
+	// initialized in Read() operation as needed.
+	var key []byte
+	var err error
+	if d.copySource {
+		if crypto.SSEC.IsEncrypted(d.metadata) {
+			d.header.Set(crypto.SSECopyKey, d.customerKeyHeader)
+			key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
+		}
+	} else {
+		if crypto.SSEC.IsEncrypted(d.metadata) {
+			d.header.Set(crypto.SSECKey, d.customerKeyHeader)
+			key, err = ParseSSECustomerHeader(d.header)
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	objectEncryptionKey, err := decryptObjectInfo(key, d.bucket, d.object, m)
+	if err != nil {
+		return err
+	}
+
+	var partIDbin [4]byte
+	binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
+
+	mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
+	mac.Write(partIDbin[:])
+	partEncryptionKey := mac.Sum(nil)
+
+	// Limit the reader, so the decryptor doesnt receive bytes
+	// from the next part (different DARE stream)
+	encLenToRead := d.parts[d.partIndex].Size - d.partEncRelOffset
+	decrypter, err := newDecryptReaderWithObjectKey(io.LimitReader(d.reader, encLenToRead), partEncryptionKey, d.startSeqNum, m)
+	if err != nil {
+		return err
+	}
+
+	d.decrypter = decrypter
+	return nil
+}
+
+func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
+	var err error
+	var n1 int
+	decPartSize, _ := sio.DecryptedSize(uint64(d.parts[d.partIndex].Size))
+	unreadPartLen := int64(decPartSize) - d.partDecRelOffset
+	if int64(len(p)) < unreadPartLen {
+		n1, err = d.decrypter.Read(p)
+		if err != nil {
+			return 0, err
+		}
+		d.partDecRelOffset += int64(n1)
+	} else {
+		n1, err = io.ReadFull(d.decrypter, p[:unreadPartLen])
+		if err != nil {
+			return 0, err
+		}
+
+		// We should now proceed to next part, reset all
+		// values appropriately.
+		d.partEncRelOffset = 0
+		d.partDecRelOffset = 0
+		d.startSeqNum = 0
+
+		d.partIndex++
+		if d.partIndex == len(d.parts) {
+			return n1, io.EOF
+		}
+
+		err = d.buildDecrypter(d.parts[d.partIndex].Number)
+		if err != nil {
+			return 0, err
+		}
+
+		n1, err = d.decrypter.Read(p[n1:])
+		if err != nil {
+			return 0, err
+		}
+
+		d.partDecRelOffset += int64(n1)
+	}
+
+	return len(p), nil
+}
+
+// DecryptBlocksWriter - decrypts  multipart parts, while implementing
+// a io.Writer compatible interface.
 type DecryptBlocksWriter struct {
 	// Original writer where the plain data will be written
 	writer io.Writer
@@ -367,7 +597,7 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 	if w.copySource {
 		if crypto.SSEC.IsEncrypted(w.metadata) {
 			w.req.Header.Set(crypto.SSECopyKey, w.customerKeyHeader)
-			key, err = ParseSSECopyCustomerRequest(w.req, w.metadata)
+			key, err = ParseSSECopyCustomerRequest(w.req.Header, w.metadata)
 		}
 	} else {
 		if crypto.SSEC.IsEncrypted(w.metadata) {
@@ -391,13 +621,6 @@ func (w *DecryptBlocksWriter) buildDecrypter(partID int) error {
 	mac.Write(partIDbin[:])
 	partEncryptionKey := mac.Sum(nil)
 
-	// make sure we do not save the key by accident
-	if w.copySource {
-		delete(m, crypto.SSECopyKey)
-	} else {
-		delete(m, crypto.SSECKey)
-	}
-
 	// make sure to provide a NopCloser such that a Close
 	// on sio.decryptWriter doesn't close the underlying writer's
 	// close which perhaps can close the stream prematurely.
@@ -666,6 +889,119 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
 	return size, nil
 }
 
+// GetDecryptedRange - To decrypt the range (off, length) of the
+// decrypted object stream, we need to read the range (encOff,
+// encLength) of the encrypted object stream to decrypt it, and
+// compute skipLen, the number of bytes to skip in the beginning of
+// the encrypted range.
+//
+// In addition we also compute the object part number for where the
+// requested range starts, along with the DARE sequence number within
+// that part. For single part objects, the partStart will be 0.
+func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, skipLen int64, seqNumber uint32, partStart int, err error) {
+	if !crypto.IsEncrypted(o.UserDefined) {
+		err = errors.New("Object is not encrypted")
+		return
+	}
+
+	if rs == nil {
+		// No range, so offsets refer to the whole object.
+		return 0, int64(o.Size), 0, 0, 0, nil
+	}
+
+	// Assemble slice of (decrypted) part sizes in `sizes`
+	var decObjSize int64 // decrypted total object size
+	var partSize uint64
+	partSize, err = sio.DecryptedSize(uint64(o.Size))
+	if err != nil {
+		return
+	}
+	sizes := []int64{int64(partSize)}
+	decObjSize = sizes[0]
+	if crypto.IsMultiPart(o.UserDefined) {
+		sizes = make([]int64, len(o.Parts))
+		decObjSize = 0
+		for i, part := range o.Parts {
+			partSize, err = sio.DecryptedSize(uint64(part.Size))
+			if err != nil {
+				return
+			}
+			t := int64(partSize)
+			sizes[i] = t
+			decObjSize += t
+		}
+	}
+
+	var off, length int64
+	off, length, err = rs.GetOffsetLength(decObjSize)
+	if err != nil {
+		return
+	}
+
+	// At this point, we have:
+	//
+	// 1. the decrypted part sizes in `sizes` (single element for
+	//    single part object) and total decrypted object size `decObjSize`
+	//
+	// 2. the (decrypted) start offset `off` and (decrypted)
+	//    length to read `length`
+	//
+	// These are the inputs to the rest of the algorithm below.
+
+	// Locate the part containing the start of the required range
+	var partEnd int
+	var cumulativeSum, encCumulativeSum int64
+	for i, size := range sizes {
+		if off < cumulativeSum+size {
+			partStart = i
+			break
+		}
+		cumulativeSum += size
+		encPartSize, _ := sio.EncryptedSize(uint64(size))
+		encCumulativeSum += int64(encPartSize)
+	}
+	// partStart is always found in the loop above,
+	// because off is validated.
+
+	sseDAREEncPackageBlockSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
+	startPkgNum := (off - cumulativeSum) / sseDAREPackageBlockSize
+
+	// Now we can calculate the number of bytes to skip
+	skipLen = (off - cumulativeSum) % sseDAREPackageBlockSize
+
+	encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize
+	// Locate the part containing the end of the required range
+	endOffset := off + length - 1
+	for i1, size := range sizes[partStart:] {
+		i := partStart + i1
+		if endOffset < cumulativeSum+size {
+			partEnd = i
+			break
+		}
+		cumulativeSum += size
+		encPartSize, _ := sio.EncryptedSize(uint64(size))
+		encCumulativeSum += int64(encPartSize)
+	}
+	// partEnd is always found in the loop above, because off and
+	// length are validated.
+	endPkgNum := (endOffset - cumulativeSum) / sseDAREPackageBlockSize
+	// Compute endEncOffset with one additional DARE package (so
+	// we read the package containing the last desired byte).
+	endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize
+	// Check if the DARE package containing the end offset is a
+	// full sized package (as the last package in the part may be
+	// smaller)
+	lastPartSize, _ := sio.EncryptedSize(uint64(sizes[partEnd]))
+	if endEncOffset > encCumulativeSum+int64(lastPartSize) {
+		endEncOffset = encCumulativeSum + int64(lastPartSize)
+	}
+	encLength = endEncOffset - encOff
+	// Set the sequence number as the starting package number of
+	// the requested block
+	seqNumber = uint32(startPkgNum)
+	return encOff, encLength, skipLen, seqNumber, partStart, nil
+}
+
 // EncryptedSize returns the size of the object after encryption.
 // An encrypted object is always larger than a plain object
 // except for zero size objects.
@@ -716,7 +1052,7 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
 // decryption succeeded.
 //
 // DecryptObjectInfo also returns whether the object is encrypted or not.
-func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, err error) {
+func DecryptObjectInfo(info ObjectInfo, headers http.Header) (encrypted bool, err error) {
 	// Directories are never encrypted.
 	if info.IsDir {
 		return false, nil
@@ -734,7 +1070,7 @@ func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, e
 			err = errEncryptedObject
 			return
 		}
-		info.Size, err = info.DecryptedSize()
+		_, err = info.DecryptedSize()
 	}
 	return
 }

cmd/encryption-v1_test.go

@@ -22,7 +22,9 @@ import (
 	"net/http"
 	"testing"
 
+	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/cmd/crypto"
+	"github.com/minio/sio"
 )
 
 var hasServerSideEncryptionHeaderTests = []struct {
@@ -208,11 +210,6 @@ func TestParseSSECustomerRequest(t *testing.T) {
 		if err != test.err {
 			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
 		}
-		key := request.Header.Get(crypto.SSECKey)
-		if (err == nil || err == crypto.ErrCustomerKeyMD5Mismatch) && key != "" {
-			t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
-		}
-
 	}
 }
 
@@ -325,14 +322,10 @@ func TestParseSSECopyCustomerRequest(t *testing.T) {
 		request.Header = headers
 		globalIsSSL = test.useTLS
 
-		_, err := ParseSSECopyCustomerRequest(request, test.metadata)
+		_, err := ParseSSECopyCustomerRequest(request.Header, test.metadata)
 		if err != test.err {
 			t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
 		}
-		key := request.Header.Get(crypto.SSECopyKey)
-		if (err == nil || err == crypto.ErrCustomerKeyMD5Mismatch) && key != "" {
-			t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
-		}
 	}
 }
 
@@ -374,9 +367,6 @@ func TestEncryptRequest(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
 		}
-		if key, ok := test.metadata[crypto.SSECKey]; ok {
-			t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
-		}
 		if kdf, ok := test.metadata[crypto.SSESealAlgorithm]; !ok {
 			t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
 		}
@@ -557,7 +547,7 @@ var decryptObjectInfoTests = []struct {
 
 func TestDecryptObjectInfo(t *testing.T) {
 	for i, test := range decryptObjectInfoTests {
-		if encrypted, err := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
+		if encrypted, err := DecryptObjectInfo(test.info, test.headers); err != test.expErr {
 			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
 		} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
 			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
@@ -566,3 +556,285 @@ func TestDecryptObjectInfo(t *testing.T) {
 		}
 	}
 }
+
+func TestGetDecryptedRange(t *testing.T) {
+	var (
+		pkgSz     = int64(64) * humanize.KiByte
+		minPartSz = int64(5) * humanize.MiByte
+		maxPartSz = int64(5) * humanize.GiByte
+
+		getEncSize = func(s int64) int64 {
+			v, _ := sio.EncryptedSize(uint64(s))
+			return int64(v)
+		}
+		udMap = func(isMulti bool) map[string]string {
+			m := map[string]string{
+				crypto.SSESealAlgorithm: SSESealAlgorithmDareSha256,
+				crypto.SSEMultipart:     "1",
+			}
+			if !isMulti {
+				delete(m, crypto.SSEMultipart)
+			}
+			return m
+		}
+	)
+
+	// Single part object tests
+	var (
+		mkSPObj = func(s int64) ObjectInfo {
+			return ObjectInfo{
+				Size:        getEncSize(s),
+				UserDefined: udMap(false),
+			}
+		}
+	)
+
+	testSP := []struct {
+		decSz int64
+		oi    ObjectInfo
+	}{
+		{0, mkSPObj(0)},
+		{1, mkSPObj(1)},
+		{pkgSz - 1, mkSPObj(pkgSz - 1)},
+		{pkgSz, mkSPObj(pkgSz)},
+		{2*pkgSz - 1, mkSPObj(2*pkgSz - 1)},
+		{minPartSz, mkSPObj(minPartSz)},
+		{maxPartSz, mkSPObj(maxPartSz)},
+	}
+
+	for i, test := range testSP {
+		{
+			// nil range
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != getEncSize(test.decSz) {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		if test.decSz >= 10 {
+			// first 10 bytes
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 0, 9})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			var rLen = pkgSz + 32
+			if test.decSz < pkgSz {
+				rLen = test.decSz + 32
+			}
+			if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != rLen {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		kb32 := int64(32) * humanize.KiByte
+		if test.decSz >= (64+32)*humanize.KiByte {
+			// Skip the first 32Kib, and read the next 64Kib
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, kb32, 3*kb32 - 1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			var rLen = (pkgSz + 32) * 2
+			if test.decSz < 2*pkgSz {
+				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32)
+			}
+			if skip != kb32 || sn != 0 || ps != 0 || o != 0 || l != rLen {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		if test.decSz >= (64*2+32)*humanize.KiByte {
+			// Skip the first 96Kib and read the next 64Kib
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 3 * kb32, 5*kb32 - 1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			var rLen = (pkgSz + 32) * 2
+			if test.decSz-pkgSz < 2*pkgSz {
+				rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32*2)
+			}
+			if skip != kb32 || sn != 1 || ps != 0 || o != pkgSz+32 || l != rLen {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+	}
+
+	// Multipart object tests
+	var (
+		// make a multipart object-info given part sizes
+		mkMPObj = func(sizes []int64) ObjectInfo {
+			r := make([]objectPartInfo, len(sizes))
+			sum := int64(0)
+			for i, s := range sizes {
+				r[i].Number = i
+				r[i].Size = int64(getEncSize(s))
+				sum += r[i].Size
+			}
+			return ObjectInfo{
+				Size:        sum,
+				UserDefined: udMap(true),
+				Parts:       r,
+			}
+		}
+		// Simple useful utilities
+		repeat = func(k int64, n int) []int64 {
+			a := []int64{}
+			for i := 0; i < n; i++ {
+				a = append(a, k)
+			}
+			return a
+		}
+		lsum = func(s []int64) int64 {
+			sum := int64(0)
+			for _, i := range s {
+				if i < 0 {
+					return -1
+				}
+				sum += i
+			}
+			return sum
+		}
+		esum = func(oi ObjectInfo) int64 {
+			sum := int64(0)
+			for _, i := range oi.Parts {
+				sum += i.Size
+			}
+			return sum
+		}
+	)
+
+	s1 := []int64{5487701, 5487799, 3}
+	s2 := repeat(5487701, 5)
+	s3 := repeat(maxPartSz, 10000)
+	testMPs := []struct {
+		decSizes []int64
+		oi       ObjectInfo
+	}{
+		{s1, mkMPObj(s1)},
+		{s2, mkMPObj(s2)},
+		{s3, mkMPObj(s3)},
+	}
+
+	// This function is a reference (re-)implementation of
+	// decrypted range computation, written solely for the purpose
+	// of the unit tests.
+	//
+	// `s` gives the decrypted part sizes, and the other
+	// parameters describe the desired read segment. When
+	// `isFromEnd` is true, `skipLen` argument is ignored.
+	decryptedRangeRef := func(s []int64, skipLen, readLen int64, isFromEnd bool) (o, l, skip int64, sn uint32, ps int) {
+		oSize := lsum(s)
+		if isFromEnd {
+			skipLen = oSize - readLen
+		}
+		if skipLen < 0 || readLen < 0 || oSize < 0 || skipLen+readLen > oSize {
+			t.Fatalf("Impossible read specified: %d %d %d", skipLen, readLen, oSize)
+		}
+
+		var cumulativeSum, cumulativeEncSum int64
+		toRead := readLen
+		readStart := false
+		for i, v := range s {
+			partOffset := int64(0)
+			partDarePkgOffset := int64(0)
+			if !readStart && cumulativeSum+v > skipLen {
+				// Read starts at the current part
+				readStart = true
+
+				partOffset = skipLen - cumulativeSum
+
+				// All return values except `l` are
+				// calculated here.
+				sn = uint32(partOffset / pkgSz)
+				skip = partOffset % pkgSz
+				ps = i
+				o = cumulativeEncSum + int64(sn)*(pkgSz+32)
+
+				partDarePkgOffset = partOffset - skip
+			}
+			if readStart {
+				currentPartBytes := v - partOffset
+				currentPartDareBytes := v - partDarePkgOffset
+				if currentPartBytes < toRead {
+					toRead -= currentPartBytes
+					l += getEncSize(currentPartDareBytes)
+				} else {
+					// current part has the last
+					// byte required
+					lbPartOffset := partOffset + toRead - 1
+
+					// round up the lbPartOffset
+					// to the end of the
+					// corresponding DARE package
+					lbPkgEndOffset := lbPartOffset - (lbPartOffset % pkgSz) + pkgSz
+					if lbPkgEndOffset > v {
+						lbPkgEndOffset = v
+					}
+					bytesToDrop := v - lbPkgEndOffset
+
+					// Last segment to update `l`
+					l += getEncSize(currentPartDareBytes - bytesToDrop)
+					break
+				}
+			}
+
+			cumulativeSum += v
+			cumulativeEncSum += getEncSize(v)
+		}
+		return
+	}
+
+	for i, test := range testMPs {
+		{
+			// nil range
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil)
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+			if o != 0 || l != esum(test.oi) || skip != 0 || sn != 0 || ps != 0 {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps)
+			}
+		}
+
+		// Skip 1Mib and read 1Mib (in the decrypted object)
+		//
+		// The check below ensures the object is large enough
+		// for the read.
+		if lsum(test.decSizes) >= 2*humanize.MiByte {
+			skipLen, readLen := int64(1)*humanize.MiByte, int64(1)*humanize.MiByte
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, skipLen, skipLen + readLen - 1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+
+			oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, skipLen, readLen, false)
+			if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
+					i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
+			}
+		}
+
+		// Read the last 6Mib+1 bytes of the (decrypted)
+		// object
+		//
+		// The check below ensures the object is large enough
+		// for the read.
+		readLen := int64(6)*humanize.MiByte + 1
+		if lsum(test.decSizes) >= readLen {
+			o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{true, -readLen, -1})
+			if err != nil {
+				t.Errorf("Case %d: unexpected err: %v", i, err)
+			}
+
+			oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, 0, readLen, true)
+			if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef {
+				t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)",
+					i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef)
+			}
+		}
+
+	}
+}

cmd/endpoint.go

@@ -168,6 +168,18 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 // EndpointList - list of same type of endpoint.
 type EndpointList []Endpoint
 
+// Nodes - returns number of unique servers.
+func (endpoints EndpointList) Nodes() int {
+	uniqueNodes := set.NewStringSet()
+	for _, endpoint := range endpoints {
+		if uniqueNodes.Contains(endpoint.Host) {
+			continue
+		}
+		uniqueNodes.Add(endpoint.Host)
+	}
+	return len(uniqueNodes)
+}
+
 // IsHTTPS - returns true if secure for URLEndpointType.
 func (endpoints EndpointList) IsHTTPS() bool {
 	return endpoints[0].IsHTTPS()

cmd/erasure-encode_test.go

@@ -28,6 +28,10 @@ import (
 
 type badDisk struct{ StorageAPI }
 
+func (a badDisk) String() string {
+	return "bad-disk"
+}
+
 func (a badDisk) AppendFile(volume string, path string, buf []byte) error {
 	return errFaultyDisk
 }

cmd/erasure-utils.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"io"
+	"strings"
 
 	"github.com/klauspost/reedsolomon"
 	"github.com/minio/minio/cmd/logger"
@@ -90,7 +91,10 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		// Copy the block.
 		n, err := io.Copy(dst, bytes.NewReader(block))
 		if err != nil {
-			logger.LogIf(ctx, err)
+			// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
+			if !strings.Contains(err.Error(), "read/write on closed pipe") {
+				logger.LogIf(ctx, err)
+			}
 			return 0, err
 		}
 

cmd/fs-v1-metadata.go

@@ -24,7 +24,6 @@ import (
 	"io/ioutil"
 	"os"
 	pathutil "path"
-	"strings"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
@@ -137,11 +136,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 
 	// Guess content-type from the extension if possible.
 	if m.Meta["content-type"] == "" {
-		if objectExt := pathutil.Ext(object); objectExt != "" {
-			if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
-				m.Meta["content-type"] = content.ContentType
-			}
-		}
+		m.Meta["content-type"] = mimedb.TypeByExtension(pathutil.Ext(object))
 	}
 
 	if hasSuffix(object, slashSeparator) {
@@ -223,11 +218,13 @@ func parseFSPartsArray(fsMetaBuf []byte) []objectPartInfo {
 		name := gjson.Get(partJSON, "name").String()
 		etag := gjson.Get(partJSON, "etag").String()
 		size := gjson.Get(partJSON, "size").Int()
+		actualSize := gjson.Get(partJSON, "actualSize").Int()
 		partsArray = append(partsArray, objectPartInfo{
-			Number: int(number),
-			Name:   name,
-			ETag:   etag,
-			Size:   size,
+			Number:     int(number),
+			Name:       name,
+			ETag:       etag,
+			Size:       size,
+			ActualSize: int64(actualSize),
 		})
 		return true
 	})

cmd/fs-v1-multipart.go

@@ -46,21 +46,25 @@ func (fs *FSObjects) getMultipartSHADir(bucket, object string) string {
 }
 
 // Returns partNumber.etag
-func (fs *FSObjects) encodePartFile(partNumber int, etag string) string {
-	return fmt.Sprintf("%.5d.%s", partNumber, etag)
+func (fs *FSObjects) encodePartFile(partNumber int, etag string, actualSize int64) string {
+	return fmt.Sprintf("%.5d.%s.%d", partNumber, etag, actualSize)
 }
 
 // Returns partNumber and etag
-func (fs *FSObjects) decodePartFile(name string) (partNumber int, etag string, err error) {
+func (fs *FSObjects) decodePartFile(name string) (partNumber int, etag string, actualSize int64, err error) {
 	result := strings.Split(name, ".")
-	if len(result) != 2 {
-		return 0, "", errUnexpected
+	if len(result) != 3 {
+		return 0, "", 0, errUnexpected
 	}
 	partNumber, err = strconv.Atoi(result[0])
 	if err != nil {
-		return 0, "", errUnexpected
+		return 0, "", 0, errUnexpected
 	}
-	return partNumber, result[1], nil
+	actualSize, err = strconv.ParseInt(result[2], 10, 64)
+	if err != nil {
+		return 0, "", 0, errUnexpected
+	}
+	return partNumber, result[1], actualSize, nil
 }
 
 // Appends parts to an appendFile sequentially.
@@ -95,7 +99,7 @@ func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploa
 		if entry == fs.metaJSONFile {
 			continue
 		}
-		partNumber, etag, err := fs.decodePartFile(entry)
+		partNumber, etag, actualSize, err := fs.decodePartFile(entry)
 		if err != nil {
 			logger.GetReqInfo(ctx).AppendTags("entry", entry)
 			logger.LogIf(ctx, err)
@@ -119,7 +123,7 @@ func (fs *FSObjects) backgroundAppend(ctx context.Context, bucket, object, uploa
 			return
 		}
 
-		file.parts = append(file.parts, PartInfo{PartNumber: nextPartNumber, ETag: etag})
+		file.parts = append(file.parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
 		nextPartNumber++
 	}
 }
@@ -255,22 +259,6 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
 		return pi, toObjectErr(err)
 	}
 
-	// Initialize pipe.
-	go func() {
-		if gerr := fs.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil {
-			if gerr = srcInfo.Writer.Close(); gerr != nil {
-				logger.LogIf(ctx, gerr)
-				return
-			}
-			return
-		}
-		// Close writer explicitly signaling we wrote all data.
-		if gerr := srcInfo.Writer.Close(); gerr != nil {
-			logger.LogIf(ctx, gerr)
-			return
-		}
-	}()
-
 	partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader, dstOpts)
 	if err != nil {
 		return pi, toObjectErr(err, dstBucket, dstObject)
@@ -292,8 +280,8 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		return pi, toObjectErr(err, bucket)
 	}
 
-	// Validate input data size and it can never be less than zero.
-	if data.Size() < 0 {
+	// Validate input data size and it can never be less than -1.
+	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return pi, toObjectErr(errInvalidArgument)
 	}
@@ -338,7 +326,7 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	if etag == "" {
 		etag = GenETag()
 	}
-	partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag))
+	partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag, data.ActualSize()))
 
 	if err = fsRenameFile(ctx, tmpPartPath, partPath); err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
@@ -355,6 +343,7 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		LastModified: fi.ModTime(),
 		ETag:         etag,
 		Size:         fi.Size(),
+		ActualSize:   data.ActualSize(),
 	}, nil
 }
 
@@ -381,8 +370,7 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 	}
 
 	uploadIDDir := fs.getUploadIDDir(bucket, object, uploadID)
-	_, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile))
-	if err != nil {
+	if _, err := fsStatFile(ctx, pathJoin(uploadIDDir, fs.metaJSONFile)); err != nil {
 		if err == errFileNotFound || err == errFileAccessDenied {
 			return result, InvalidUploadID{UploadID: uploadID}
 		}
@@ -400,7 +388,7 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 		if entry == fs.metaJSONFile {
 			continue
 		}
-		partNumber, etag1, derr := fs.decodePartFile(entry)
+		partNumber, etag1, _, derr := fs.decodePartFile(entry)
 		if derr != nil {
 			logger.LogIf(ctx, derr)
 			return result, toObjectErr(derr)
@@ -410,11 +398,11 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 			partsMap[partNumber] = etag1
 			continue
 		}
-		stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag1)))
+		stat1, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag1)))
 		if serr != nil {
 			return result, toObjectErr(serr)
 		}
-		stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(partNumber, etag2)))
+		stat2, serr := fsStatFile(ctx, pathJoin(uploadIDDir, getPartFile(entries, partNumber, etag2)))
 		if serr != nil {
 			return result, toObjectErr(serr)
 		}
@@ -424,8 +412,19 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 	}
 
 	var parts []PartInfo
+	var actualSize int64
 	for partNumber, etag := range partsMap {
-		parts = append(parts, PartInfo{PartNumber: partNumber, ETag: etag})
+		partFile := getPartFile(entries, partNumber, etag)
+		if partFile == "" {
+			return result, InvalidPart{}
+		}
+		// Read the actualSize from the pathFileName.
+		subParts := strings.Split(partFile, ".")
+		actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
+		if err != nil {
+			return result, InvalidPart{}
+		}
+		parts = append(parts, PartInfo{PartNumber: partNumber, ETag: etag, ActualSize: actualSize})
 	}
 	sort.Slice(parts, func(i int, j int) bool {
 		return parts[i].PartNumber < parts[j].PartNumber
@@ -455,12 +454,12 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 	}
 	for i, part := range result.Parts {
 		var stat os.FileInfo
-		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag)))
+		stat, err = fsStatFile(ctx, pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag, part.ActualSize)))
 		if err != nil {
 			return result, toObjectErr(err)
 		}
 		result.Parts[i].LastModified = stat.ModTime()
-		result.Parts[i].Size = stat.Size()
+		result.Parts[i].Size = part.ActualSize
 	}
 
 	fsMetaBytes, err := ioutil.ReadFile(pathJoin(uploadIDDir, fs.metaJSONFile))
@@ -480,6 +479,9 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
 //
 // Implements S3 compatible Complete multipart API.
 func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) {
+
+	var actualSize int64
+
 	if err := checkCompleteMultipartArgs(ctx, bucket, object, fs); err != nil {
 		return oi, toObjectErr(err)
 	}
@@ -516,9 +518,37 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	// Allocate parts similar to incoming slice.
 	fsMeta.Parts = make([]objectPartInfo, len(parts))
 
+	entries, err := readDir(uploadIDDir)
+	if err != nil {
+		logger.GetReqInfo(ctx).AppendTags("uploadIDDir", uploadIDDir)
+		logger.LogIf(ctx, err)
+		return oi, err
+	}
+
+	// Save consolidated actual size.
+	var objectActualSize int64
 	// Validate all parts and then commit to disk.
 	for i, part := range parts {
-		partPath := pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag))
+		partFile := getPartFile(entries, part.PartNumber, part.ETag)
+		if partFile == "" {
+			return oi, InvalidPart{
+				PartNumber: part.PartNumber,
+				GotETag:    part.ETag,
+			}
+		}
+
+		// Read the actualSize from the pathFileName.
+		subParts := strings.Split(partFile, ".")
+		actualSize, err = strconv.ParseInt(subParts[len(subParts)-1], 10, 64)
+		if err != nil {
+			return oi, InvalidPart{
+				PartNumber: part.PartNumber,
+				GotETag:    part.ETag,
+			}
+		}
+
+		partPath := pathJoin(uploadIDDir, partFile)
+
 		var fi os.FileInfo
 		fi, err = fsStatFile(ctx, partPath)
 		if err != nil {
@@ -528,24 +558,28 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 			return oi, err
 		}
 		if partSize == -1 {
-			partSize = fi.Size()
+			partSize = actualSize
 		}
 
 		fsMeta.Parts[i] = objectPartInfo{
-			Number: part.PartNumber,
-			ETag:   part.ETag,
-			Size:   fi.Size(),
+			Number:     part.PartNumber,
+			ETag:       part.ETag,
+			Size:       fi.Size(),
+			ActualSize: actualSize,
 		}
 
+		// Consolidate the actual size.
+		objectActualSize += actualSize
+
 		if i == len(parts)-1 {
 			break
 		}
 
 		// All parts except the last part has to be atleast 5MB.
-		if !isMinAllowedPartSize(fi.Size()) {
+		if !isMinAllowedPartSize(actualSize) {
 			return oi, PartTooSmall{
 				PartNumber: part.PartNumber,
-				PartSize:   fi.Size(),
+				PartSize:   actualSize,
 				PartETag:   part.ETag,
 			}
 		}
@@ -587,11 +621,12 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	if appendFallback {
-		fsRemoveFile(ctx, file.filePath)
+		if file != nil {
+			fsRemoveFile(ctx, file.filePath)
+		}
 		for _, part := range parts {
-			partPath := pathJoin(uploadIDDir, fs.encodePartFile(part.PartNumber, part.ETag))
-			err = mioutil.AppendFile(appendFilePath, partPath)
-			if err != nil {
+			partPath := getPartFile(entries, part.PartNumber, part.ETag)
+			if err = mioutil.AppendFile(appendFilePath, pathJoin(uploadIDDir, partPath)); err != nil {
 				logger.LogIf(ctx, err)
 				return oi, toObjectErr(err)
 			}
@@ -628,6 +663,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 		fsMeta.Meta = make(map[string]string)
 	}
 	fsMeta.Meta["etag"] = s3MD5
+	// Save consolidated actual size.
+	fsMeta.Meta[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 	if _, err = fsMeta.WriteTo(metaFile); err != nil {
 		logger.LogIf(ctx, err)
 		return oi, toObjectErr(err, bucket, object)

cmd/fs-v1.go

@@ -17,10 +17,12 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"encoding/hex"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os"
 	"path"
 	"sort"
@@ -416,35 +418,19 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string) error {
 // update metadata.
 func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, e error) {
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
-	// Hold write lock on destination since in both cases
-	// - if source and destination are same
-	// - if source and destination are different
-	// it is the sole mutating state.
-	objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
-	if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
-		return oi, err
-	}
-	defer objectDWLock.Unlock()
-	// if source and destination are different, we have to hold
-	// additional read lock as well to protect against writes on
-	// source.
 	if !cpSrcDstSame {
-		// Hold read locks on source object only if we are
-		// going to read data from source object.
-		objectSRLock := fs.nsMutex.NewNSLock(srcBucket, srcObject)
-		if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
+		objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
+		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return oi, err
 		}
-		defer objectSRLock.RUnlock()
+		defer objectDWLock.Unlock()
 	}
+
 	if _, err := fs.statBucketDir(ctx, srcBucket); err != nil {
 		return oi, toObjectErr(err, srcBucket)
 	}
 
 	if cpSrcDstSame && srcInfo.metadataOnly {
-		// Close any writer which was initialized.
-		defer srcInfo.Writer.Close()
-
 		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, srcBucket, srcObject, fs.metaJSONFile)
 		wlk, err := fs.rwPool.Write(fsMetaPath)
 		if err != nil {
@@ -476,20 +462,6 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
 
-	go func() {
-		if gerr := fs.getObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, !cpSrcDstSame); gerr != nil {
-			if gerr = srcInfo.Writer.Close(); gerr != nil {
-				logger.LogIf(ctx, gerr)
-			}
-			return
-		}
-		// Close writer explicitly signaling we wrote all data.
-		if gerr := srcInfo.Writer.Close(); gerr != nil {
-			logger.LogIf(ctx, gerr)
-			return
-		}
-	}()
-
 	objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.Reader, srcInfo.UserDefined)
 	if err != nil {
 		return oi, toObjectErr(err, dstBucket, dstObject)
@@ -498,6 +470,100 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 	return objInfo, nil
 }
 
+// GetObjectNInfo - returns object info and a reader for object
+// content.
+func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+
+	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
+		return nil, err
+	}
+
+	if _, err = fs.statBucketDir(ctx, bucket); err != nil {
+		return nil, toObjectErr(err, bucket)
+	}
+
+	var nsUnlocker = func() {}
+
+	if lockType != noLock {
+		// Lock the object before reading.
+		lock := fs.nsMutex.NewNSLock(bucket, object)
+		switch lockType {
+		case writeLock:
+			if err = lock.GetLock(globalObjectTimeout); err != nil {
+				logger.LogIf(ctx, err)
+				return nil, err
+			}
+			nsUnlocker = lock.Unlock
+		case readLock:
+			if err = lock.GetRLock(globalObjectTimeout); err != nil {
+				logger.LogIf(ctx, err)
+				return nil, err
+			}
+			nsUnlocker = lock.RUnlock
+		}
+	}
+
+	// For a directory, we need to send an reader that returns no bytes.
+	if hasSuffix(object, slashSeparator) {
+		// The lock taken above is released when
+		// objReader.Close() is called by the caller.
+		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), ObjectInfo{}, nsUnlocker), nil
+	}
+
+	// Otherwise we get the object info
+	var objInfo ObjectInfo
+	if objInfo, err = fs.getObjectInfo(ctx, bucket, object); err != nil {
+		nsUnlocker()
+		return nil, toObjectErr(err, bucket, object)
+	}
+
+	// Take a rwPool lock for NFS gateway type deployment
+	rwPoolUnlocker := func() {}
+	if bucket != minioMetaBucket && lockType != noLock {
+		fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+		_, err = fs.rwPool.Open(fsMetaPath)
+		if err != nil && err != errFileNotFound {
+			logger.LogIf(ctx, err)
+			nsUnlocker()
+			return nil, toObjectErr(err, bucket, object)
+		}
+		// Need to clean up lock after getObject is
+		// completed.
+		rwPoolUnlocker = func() { fs.rwPool.Close(fsMetaPath) }
+	}
+
+	objReaderFn, off, length, rErr := NewGetObjectReader(rs, objInfo, nsUnlocker, rwPoolUnlocker)
+	if rErr != nil {
+		return nil, rErr
+	}
+
+	// Read the object, doesn't exist returns an s3 compatible error.
+	fsObjPath := pathJoin(fs.fsPath, bucket, object)
+	readCloser, size, err := fsOpenFile(ctx, fsObjPath, off)
+	if err != nil {
+		rwPoolUnlocker()
+		nsUnlocker()
+		return nil, toObjectErr(err, bucket, object)
+	}
+	var reader io.Reader
+	reader = io.LimitReader(readCloser, length)
+	closeFn := func() {
+		readCloser.Close()
+	}
+
+	// Check if range is valid
+	if off > size || off+length > size {
+		err = InvalidRange{off, length, size}
+		logger.LogIf(ctx, err)
+		closeFn()
+		rwPoolUnlocker()
+		nsUnlocker()
+		return nil, err
+	}
+
+	return objReaderFn(reader, h, closeFn)
+}
+
 // GetObject - reads an object from the disk.
 // Supports additional parameters like offset and length
 // which are synonymous with HTTP Range requests.
@@ -596,7 +662,10 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
 	buf := make([]byte, int(bufSize))
 
 	_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
-	logger.LogIf(ctx, err)
+	// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
+	if err == io.ErrClosedPipe {
+		err = nil
+	}
 	return toObjectErr(err, bucket, object)
 }
 
@@ -605,13 +674,8 @@ func (fs *FSObjects) createFsJSON(object, fsMetaPath string) error {
 	fsMeta := newFSMetaV1()
 	fsMeta.Meta = make(map[string]string)
 	fsMeta.Meta["etag"] = GenETag()
-	if objectExt := path.Ext(object); objectExt != "" {
-		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
-			fsMeta.Meta["content-type"] = content.ContentType
-		} else {
-			fsMeta.Meta["content-type"] = "application/octet-stream"
-		}
-	}
+	contentType := mimedb.TypeByExtension(path.Ext(object))
+	fsMeta.Meta["content-type"] = contentType
 	wlk, werr := fs.rwPool.Create(fsMetaPath)
 	if werr == nil {
 		_, err := fsMeta.WriteTo(wlk)
@@ -626,13 +690,8 @@ func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 {
 	fsMeta := newFSMetaV1()
 	fsMeta.Meta = make(map[string]string)
 	fsMeta.Meta["etag"] = defaultEtag
-	if objectExt := path.Ext(object); objectExt != "" {
-		if content, ok := mimedb.DB[strings.ToLower(strings.TrimPrefix(objectExt, "."))]; ok {
-			fsMeta.Meta["content-type"] = content.ContentType
-		} else {
-			fsMeta.Meta["content-type"] = "application/octet-stream"
-		}
-	}
+	contentType := mimedb.TypeByExtension(path.Ext(object))
+	fsMeta.Meta["content-type"] = contentType
 	return fsMeta
 }
 
@@ -816,7 +875,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	}
 
 	// Validate input data size and it can never be less than zero.
-	if data.Size() < 0 {
+	if data.Size() < -1 {
 		logger.LogIf(ctx, errInvalidArgument)
 		return ObjectInfo{}, errInvalidArgument
 	}
@@ -1259,3 +1318,8 @@ func (fs *FSObjects) IsNotificationSupported() bool {
 func (fs *FSObjects) IsEncryptionSupported() bool {
 	return true
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (fs *FSObjects) IsCompressionSupported() bool {
+	return true
+}

cmd/gateway-unsupported.go

@@ -151,3 +151,8 @@ func (a GatewayUnsupported) IsNotificationSupported() bool {
 func (a GatewayUnsupported) IsEncryptionSupported() bool {
 	return false
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (a GatewayUnsupported) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/azure/gateway-azure.go

@@ -615,6 +615,31 @@ func (a *azureObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 	return result, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (a *azureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = a.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := a.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject - reads an object from azure. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
@@ -660,10 +685,33 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
 
+	// Populate correct ETag's if possible, this code primarily exists
+	// because AWS S3 indicates that
+	//
+	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTCommonResponseHeaders.html
+	//
+	// Objects created by the PUT Object, POST Object, or Copy operation,
+	// or through the AWS Management Console, and are encrypted by SSE-S3
+	// or plaintext, have ETags that are an MD5 digest of their object data.
+	//
+	// Some applications depend on this behavior refer https://github.com/minio/minio/issues/6550
+	// So we handle it here and make this consistent.
+	etag := minio.ToS3ETag(blob.Properties.Etag)
+	switch {
+	case blob.Properties.ContentMD5 != "":
+		b, err := base64.StdEncoding.DecodeString(blob.Properties.ContentMD5)
+		if err == nil {
+			etag = hex.EncodeToString(b)
+		}
+	case blob.Metadata["md5sum"] != "":
+		etag = blob.Metadata["md5sum"]
+		delete(blob.Metadata, "md5sum")
+	}
+
 	return minio.ObjectInfo{
 		Bucket:          bucket,
 		UserDefined:     azurePropertiesToS3Meta(blob.Metadata, blob.Properties),
-		ETag:            minio.ToS3ETag(blob.Properties.Etag),
+		ETag:            etag,
 		ModTime:         time.Time(blob.Properties.LastModified),
 		Name:            object,
 		Size:            blob.Properties.ContentLength,
@@ -675,15 +723,103 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 // PutObject - Create a new blob with the incoming data,
 // uses Azure equivalent CreateBlockBlobFromReader.
 func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
+	if data.Size() < azureBlockSize/10 {
+		blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
+		blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+		if err = blob.CreateBlockBlobFromReader(data, nil); err != nil {
+			return objInfo, azureToObjectError(err, bucket, object)
+		}
+		return a.GetObjectInfo(ctx, bucket, object, opts)
+	}
+
+	uuid, err := getAzureUploadID()
+	if err != nil {
+		return objInfo, err
+	}
+	etag := data.MD5HexString()
+	if etag == "" {
+		etag = minio.GenETag()
+	}
+
 	blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
-	blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
+	subPartSize, subPartNumber := int64(azureBlockSize), 1
+	for remainingSize := data.Size(); remainingSize >= 0; remainingSize -= subPartSize {
+		// Allow to create zero sized part.
+		if remainingSize == 0 && subPartNumber > 1 {
+			break
+		}
+
+		if remainingSize < subPartSize {
+			subPartSize = remainingSize
+		}
+
+		id := azureGetBlockID(1, subPartNumber, uuid, etag)
+		if err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil); err != nil {
+			return objInfo, azureToObjectError(err, bucket, object)
+		}
+		subPartNumber++
+	}
+
+	objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
+	resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
-	err = blob.CreateBlockBlobFromReader(data, nil)
+
+	getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, aerr error) {
+		for _, part := range resp.UncommittedBlocks {
+			var partID int
+			var readUploadID string
+			var md5Hex string
+			if partID, _, readUploadID, md5Hex, aerr = azureParseBlockID(part.Name); aerr != nil {
+				return nil, 0, aerr
+			}
+
+			if partNumber == partID && uuid == readUploadID && etag == md5Hex {
+				blocks = append(blocks, storage.Block{
+					ID:     part.Name,
+					Status: storage.BlockStatusUncommitted,
+				})
+
+				size += part.Size
+			}
+		}
+
+		if len(blocks) == 0 {
+			return nil, 0, minio.InvalidPart{}
+		}
+
+		return blocks, size, nil
+	}
+
+	var blocks []storage.Block
+	blocks, _, err = getBlocks(1, etag)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return objInfo, err
+	}
+
+	if err = objBlob.PutBlockList(blocks, nil); err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+
+	if len(metadata) == 0 {
+		metadata = map[string]string{}
+	}
+
+	// Save md5sum for future processing on the object.
+	metadata["x-amz-meta-md5sum"] = hex.EncodeToString(data.MD5Current())
+	objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
 	if err != nil {
 		return objInfo, azureToObjectError(err, bucket, object)
 	}
+	if err = objBlob.SetProperties(nil); err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+	if err = objBlob.SetMetadata(nil); err != nil {
+		return objInfo, azureToObjectError(err, bucket, object)
+	}
+
 	return a.GetObjectInfo(ctx, bucket, object, opts)
 }
 
@@ -949,10 +1085,6 @@ func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 	}
 
 	defer func() {
-		if err != nil {
-			return
-		}
-
 		blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
 		derr := blob.Delete(nil)
 		logger.GetReqInfo(ctx).AppendTags("uploadID", uploadID)
@@ -1123,3 +1255,8 @@ func (a *azureObjects) DeleteBucketPolicy(ctx context.Context, bucket string) er
 	err := container.SetPermissions(perm, nil)
 	return azureToObjectError(err)
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (a *azureObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/b2/gateway-b2.go

@@ -23,6 +23,7 @@ import (
 	"hash"
 	"io"
 	"io/ioutil"
+	"net/http"
 
 	"strings"
 	"sync"
@@ -394,6 +395,31 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	return loi, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *b2Objects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject reads an object from B2. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
@@ -814,3 +840,8 @@ func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err)
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *b2Objects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/gcs/gateway-gcs.go

@@ -736,6 +736,31 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
 	}, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject - reads an object from GCS. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
@@ -750,7 +775,13 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s
 		return gcsToObjectError(err, bucket)
 	}
 
-	object := l.client.Bucket(bucket).Object(key)
+	// GCS storage decompresses a gzipped object by default and returns the data.
+	// Refer to https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding
+	// Need to set `Accept-Encoding` header to `gzip` when issuing a GetObject call, to be able
+	// to download the object in compressed state.
+	// Calling ReadCompressed with true accomplishes that.
+	object := l.client.Bucket(bucket).Object(key).ReadCompressed(true)
+
 	r, err := object.NewRangeReader(l.ctx, startOffset, length)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -1422,3 +1453,8 @@ func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *gcsGateway) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/manta/gateway-manta.go

@@ -506,6 +506,31 @@ func (t *tritonObjects) ListObjectsV2(ctx context.Context, bucket, prefix, conti
 	return result, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (t *tritonObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = t.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := t.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject - Reads an object from Manta. Supports additional parameters like
 // offset and length which are synonymous with HTTP Range requests.
 //
@@ -634,3 +659,8 @@ func (t *tritonObjects) DeleteObject(ctx context.Context, bucket, object string)
 
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (t *tritonObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/nas/gateway-nas.go

@@ -125,3 +125,8 @@ type nasObjects struct {
 func (l *nasObjects) IsNotificationSupported() bool {
 	return false
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *nasObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/oss/gateway-oss.go

@@ -68,7 +68,7 @@ ENVIRONMENT VARIABLES:
 
   DOMAIN:
      MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
-	
+
   CACHE:
      MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
      MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
@@ -546,6 +546,31 @@ func ossGetObject(ctx context.Context, client *oss.Client, bucket, key string, s
 	return nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *ossObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject reads an object on OSS. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
@@ -1070,3 +1095,8 @@ func (l *ossObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 	}
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *ossObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/s3/gateway-s3.go

@@ -21,11 +21,13 @@ import (
 	"encoding/json"
 	"io"
 	"math/rand"
+	"net/http"
 	"strings"
 	"time"
 
 	"github.com/minio/cli"
 	miniogo "github.com/minio/minio-go"
+	"github.com/minio/minio-go/pkg/credentials"
 	"github.com/minio/minio-go/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
@@ -88,6 +90,15 @@ EXAMPLES:
      $ export MINIO_CACHE_EXPIRY=40
      $ export MINIO_CACHE_MAXUSE=80
      $ {{.HelpName}}
+
+  4. Start minio gateway server for AWS S3 backend using AWS environment variables.
+     NOTE: The access and secret key in this case will authenticate with Minio instead
+     of AWS and AWS envs will be used to authenticate to AWS S3.
+     $ export AWS_ACCESS_KEY_ID=aws_access_key
+     $ export AWS_SECRET_ACCESS_KEY=aws_secret_key
+     $ export MINIO_ACCESS_KEY=accesskey
+     $ export MINIO_SECRET_KEY=secretkey
+     $ {{.HelpName}}
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
@@ -149,7 +160,7 @@ func randString(n int, src rand.Source, prefix string) string {
 }
 
 // newS3 - Initializes a new client by auto probing S3 server signature.
-func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
+func newS3(url string) (*miniogo.Core, error) {
 	if url == "" {
 		url = "https://s3.amazonaws.com"
 	}
@@ -160,19 +171,33 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 		return nil, err
 	}
 
-	clnt, err := miniogo.NewV4(endpoint, accessKey, secretKey, secure)
+	// Chains all credential types, in the following order:
+	//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+	//  - IAM profile based credentials. (performs an HTTP
+	//    call to a pre-defined endpoint, only valid inside
+	//    configured ec2 instances)
+	//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
+	//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
+	creds := credentials.NewChainCredentials([]credentials.Provider{
+		&credentials.EnvAWS{},
+		&credentials.IAM{
+			Client: &http.Client{
+				Transport: minio.NewCustomHTTPTransport(),
+			},
+		},
+		&credentials.FileAWSCredentials{},
+		&credentials.EnvMinio{},
+	})
+
+	clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "")
 	if err != nil {
 		return nil, err
 	}
+
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+	// Check if the provided keys are valid.
 	if _, err = clnt.BucketExists(probeBucketName); err != nil {
-		clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
-		if err != nil {
-			return nil, err
-		}
-		if _, err = clnt.BucketExists(probeBucketName); err != nil {
-			return nil, err
-		}
+		return nil, err
 	}
 
 	return &miniogo.Core{Client: clnt}, nil
@@ -180,8 +205,9 @@ func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
 
 // NewGatewayLayer returns s3 ObjectLayer.
 func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
-	// Probe S3 signature with input credentials.
-	clnt, err := newS3(g.host, creds.AccessKey, creds.SecretKey)
+	// creds are ignored here, since S3 gateway implements chaining
+	// all credentials.
+	clnt, err := newS3(g.host)
 	if err != nil {
 		return nil, err
 	}
@@ -301,6 +327,31 @@ func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	return minio.FromMinioClientListBucketV2Result(bucket, result), nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := l.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 // GetObject reads an object from S3. Supports additional
 // parameters like offset and length which are synonymous with
 // HTTP Range requests.
@@ -487,3 +538,8 @@ func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 	}
 	return nil
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (l *s3Objects) IsCompressionSupported() bool {
+	return false
+}

cmd/gateway/sia/gateway-sia.go

@@ -431,6 +431,31 @@ func (s *siaObjects) ListObjects(ctx context.Context, bucket string, prefix stri
 	return loi, nil
 }
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (s *siaObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, opts minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
+	var objInfo minio.ObjectInfo
+	objInfo, err = s.GetObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	var startOffset, length int64
+	startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
+	if err != nil {
+		return nil, err
+	}
+
+	pr, pw := io.Pipe()
+	go func() {
+		err := s.GetObject(ctx, bucket, object, startOffset, length, pw, objInfo.ETag, opts)
+		pw.CloseWithError(err)
+	}()
+	// Setup cleanup function to cause the above go-routine to
+	// exit in case of partial read
+	pipeCloser := func() { pr.Close() }
+	return minio.NewGetObjectReaderFromReader(pr, objInfo, pipeCloser), nil
+}
+
 func (s *siaObjects) GetObject(ctx context.Context, bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
 	dstFile := path.Join(s.TempDir, minio.MustGetUUID())
 	defer os.Remove(dstFile)
@@ -626,3 +651,8 @@ func (s *siaObjects) deleteTempFileWhenUploadCompletes(ctx context.Context, temp
 
 	os.Remove(tempFile)
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (s *siaObjects) IsCompressionSupported() bool {
+	return false
+}

cmd/generic-handlers.go

@@ -689,7 +689,7 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 // canceled immediately.
 func setRateLimitHandler(h http.Handler) http.Handler {
 	_, maxLimit, err := sys.GetMaxOpenFileLimit()
-	logger.FatalIf(err, "Unable to get maximum open file limit", context.Background())
+	logger.FatalIf(err, "Unable to get maximum open file limit")
 	// Burst value is set to 1 to allow only maxOpenFileLimit
 	// requests to happen at once.
 	l := rate.NewLimiter(rate.Limit(maxLimit), 1)

cmd/globals.go

@@ -227,6 +227,23 @@ var (
 	globalKMS crypto.KMS
 	// KMS config
 	globalKMSConfig crypto.KMSConfig
+
+	// Is compression include extensions/content-types set.
+	globalIsEnvCompression bool
+
+	// Is compression enabeld.
+	globalIsCompressionEnabled = false
+
+	// Include-list for compression.
+	globalCompressExtensions = []string{".txt", ".log", ".csv", ".json"}
+	globalCompressMimeTypes  = []string{"text/csv", "text/plain", "application/json"}
+
+	// Some standard object extensions which we strictly dis-allow for compression.
+	standardExcludeCompressExtensions = []string{".gz", ".bz2", ".rar", ".zip", ".7z"}
+
+	// Some standard content-types which we strictly dis-allow for compression.
+	standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"}
+
 	// Add new variable global values here.
 )
 

cmd/http/listener.go

@@ -35,6 +35,8 @@ import (
 
 var sslRequiredErrMsg = []byte("HTTP/1.0 403 Forbidden\r\n\r\nSSL required")
 
+var badRequestMsg = []byte("HTTP/1.0 400 Bad Request\r\n\r\n")
+
 // HTTP methods.
 var methods = []string{
 	http.MethodGet,
@@ -73,11 +75,40 @@ func isHTTPMethod(s string) bool {
 	return false
 }
 
-func getResourceHost(bufConn *BufConn, maxHeaderBytes, methodLen int) (resource string, host string, err error) {
+func getPlainText(bufConn *BufConn) (bool, error) {
+	defer bufConn.setReadTimeout()
+
+	if bufConn.canSetReadDeadline() {
+		// Set deadline such that we close the connection quickly
+		// of no data was received from the Peek()
+		bufConn.SetReadDeadline(time.Now().UTC().Add(time.Second * 3))
+	}
+	b, err := bufConn.Peek(1)
+	if err != nil {
+		return false, err
+	}
+	for _, method := range methods {
+		if b[0] == method[0] {
+			return true, nil
+		}
+	}
+	return false, nil
+}
+
+func getMethodResourceHost(bufConn *BufConn, maxHeaderBytes int) (method string, resource string, host string, err error) {
+	defer bufConn.setReadTimeout()
+
 	var data []byte
-	for dataLen := 0; methodLen+dataLen < maxHeaderBytes; dataLen += 8 {
-		if data, err = bufConn.Peek(methodLen + dataLen); err != nil {
-			return "", "", err
+	for dataLen := 1; dataLen < maxHeaderBytes; dataLen++ {
+		if bufConn.canSetReadDeadline() {
+			// Set deadline such that we close the connection quickly
+			// of no data was received from the Peek()
+			bufConn.SetReadDeadline(time.Now().UTC().Add(time.Second * 3))
+		}
+
+		data, err = bufConn.bufReader.Peek(dataLen)
+		if err != nil {
+			return "", "", "", err
 		}
 
 		tokens := strings.Split(string(data), "\n")
@@ -85,8 +116,21 @@ func getResourceHost(bufConn *BufConn, maxHeaderBytes, methodLen int) (resource
 			continue
 		}
 
-		if resource == "" {
-			resource = strings.SplitN(tokens[0][methodLen:], " ", 2)[0]
+		if method == "" && resource == "" {
+			if i := strings.IndexByte(tokens[0], ' '); i == -1 {
+				return "", "", "", fmt.Errorf("malformed HTTP request from '%s'", bufConn.LocalAddr())
+			}
+			httpTokens := strings.SplitN(tokens[0], " ", 3)
+			if len(httpTokens) < 3 {
+				return "", "", "", fmt.Errorf("malformed HTTP request from '%s'", bufConn.LocalAddr())
+			}
+			if !isHTTPMethod(httpTokens[0]) {
+				return "", "", "", fmt.Errorf("malformed HTTP request, invalid HTTP method '%s' from '%s'",
+					httpTokens[0], bufConn.LocalAddr())
+			}
+
+			method = httpTokens[0]
+			resource = httpTokens[1]
 		}
 
 		for _, token := range tokens[1:] {
@@ -99,7 +143,7 @@ func getResourceHost(bufConn *BufConn, maxHeaderBytes, methodLen int) (resource
 			token = strings.ToLower(token)
 			if strings.HasPrefix(token, "host: ") {
 				host = strings.TrimPrefix(strings.TrimSuffix(token, "\r"), "host: ")
-				return resource, host, nil
+				return method, resource, host, nil
 			}
 		}
 
@@ -108,7 +152,7 @@ func getResourceHost(bufConn *BufConn, maxHeaderBytes, methodLen int) (resource
 		}
 	}
 
-	return resource, host, nil
+	return "", "", "", fmt.Errorf("malformed HTTP request from %s", bufConn.LocalAddr())
 }
 
 type acceptResult struct {
@@ -175,9 +219,48 @@ func (listener *httpListener) start() {
 		tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout)
 
 		bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout)
+		if listener.tlsConfig != nil {
+			ok, err := getPlainText(bufconn)
+			if err != nil {
+				// Peek could fail legitimately when clients abruptly close
+				// connection. E.g. Chrome browser opens connections speculatively to
+				// speed up loading of a web page. Peek may also fail due to network
+				// saturation on a transport with read timeout set. All other kind of
+				// errors should be logged for further investigation. Thanks @brendanashworth.
+				if !isRoutineNetErr(err) {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
+					reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
+					ctx := logger.SetReqInfo(context.Background(), reqInfo)
+					logger.LogIf(ctx, err)
+				}
+				bufconn.Close()
+				return
+			}
 
-		// Peek bytes of maximum length of all HTTP methods.
-		data, err := bufconn.Peek(methodMaxLen)
+			if ok {
+				// As TLS is configured and we got plain text HTTP request,
+				// return 403 (forbidden) error.
+				bufconn.Write(sslRequiredErrMsg)
+				bufconn.Close()
+				return
+			}
+
+			// As the listener is configured with TLS, try to do TLS handshake, drop the connection if it fails.
+			tlsConn := tls.Server(bufconn, listener.tlsConfig)
+
+			if err := tlsConn.Handshake(); err != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
+				reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
+				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				logger.LogIf(ctx, err)
+				bufconn.Close()
+				return
+			}
+
+			bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout)
+		}
+
+		method, resource, host, err := getMethodResourceHost(bufconn, listener.maxHeaderBytes)
 		if err != nil {
 			// Peek could fail legitimately when clients abruptly close
 			// connection. E.g. Chrome browser opens connections speculatively to
@@ -189,116 +272,25 @@ func (listener *httpListener) start() {
 				reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
 				ctx := logger.SetReqInfo(context.Background(), reqInfo)
 				logger.LogIf(ctx, err)
+				bufconn.Write(badRequestMsg)
 			}
 			bufconn.Close()
 			return
 		}
 
-		// Return bufconn if read data is a valid HTTP method.
-		tokens := strings.SplitN(string(data), " ", 2)
-		if isHTTPMethod(tokens[0]) {
-			if listener.tlsConfig != nil {
-				// As TLS is configured and we got plain text HTTP request,
-				// return 403 (forbidden) error.
-				bufconn.Write(sslRequiredErrMsg)
-				bufconn.Close()
-				return
-			}
-
-			var resource, host string
-			if resource, host, err = getResourceHost(bufconn, listener.maxHeaderBytes, len(tokens[0])+1); err != nil {
-				if !isRoutineNetErr(err) {
-					reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
-					reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
-					ctx := logger.SetReqInfo(context.Background(), reqInfo)
-					logger.LogIf(ctx, err)
-				}
-				bufconn.Close()
-				return
-			}
-
-			header := make(http.Header)
-			if host != "" {
-				header.Add("Host", host)
-			}
-			bufconn.setRequest(&http.Request{
-				Method: tokens[0],
-				URL:    &url.URL{Path: resource},
-				Host:   bufconn.LocalAddr().String(),
-				Header: header,
-			})
-			bufconn.setUpdateFuncs(listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
-
-			send(acceptResult{bufconn, nil}, doneCh)
-			return
+		header := make(http.Header)
+		if host != "" {
+			header.Add("Host", host)
 		}
+		bufconn.setRequest(&http.Request{
+			Method: method,
+			URL:    &url.URL{Path: resource},
+			Host:   bufconn.LocalAddr().String(),
+			Header: header,
+		})
+		bufconn.setUpdateFuncs(listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
 
-		if listener.tlsConfig != nil {
-			// As the listener is configured with TLS, try to do TLS handshake, drop the connection if it fails.
-			tlsConn := tls.Server(bufconn, listener.tlsConfig)
-			if err = tlsConn.Handshake(); err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
-				reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
-				ctx := logger.SetReqInfo(context.Background(), reqInfo)
-				logger.LogIf(ctx, err)
-				bufconn.Close()
-				return
-			}
-
-			// Check whether the connection contains HTTP request or not.
-			bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout)
-
-			// Peek bytes of maximum length of all HTTP methods.
-			data, err = bufconn.Peek(methodMaxLen)
-			if err != nil {
-				if !isRoutineNetErr(err) {
-					reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
-					reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
-					ctx := logger.SetReqInfo(context.Background(), reqInfo)
-					logger.LogIf(ctx, err)
-				}
-				bufconn.Close()
-				return
-			}
-
-			// Return bufconn if read data is a valid HTTP method.
-			tokens := strings.SplitN(string(data), " ", 2)
-			if isHTTPMethod(tokens[0]) {
-				var resource, host string
-				if resource, host, err = getResourceHost(bufconn, listener.maxHeaderBytes, len(tokens[0])+1); err != nil {
-					if !isRoutineNetErr(err) {
-						reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
-						reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
-						ctx := logger.SetReqInfo(context.Background(), reqInfo)
-						logger.LogIf(ctx, err)
-					}
-					bufconn.Close()
-					return
-				}
-
-				header := make(http.Header)
-				if host != "" {
-					header.Add("Host", host)
-				}
-				bufconn.setRequest(&http.Request{
-					Method: tokens[0],
-					URL:    &url.URL{Path: resource},
-					Host:   bufconn.LocalAddr().String(),
-					Header: header,
-				})
-				bufconn.setUpdateFuncs(listener.updateBytesReadFunc, listener.updateBytesWrittenFunc)
-
-				send(acceptResult{bufconn, nil}, doneCh)
-				return
-			}
-		}
-		reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", bufconn.RemoteAddr().String())
-		reqInfo.AppendTags("localAddr", bufconn.LocalAddr().String())
-		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		logger.LogIf(ctx, err)
-
-		bufconn.Close()
-		return
+		send(acceptResult{bufconn, nil}, doneCh)
 	}
 
 	// Closure to handle TCPListener until done channel is closed.

cmd/http/listener_test.go

@@ -393,11 +393,11 @@ func TestHTTPListenerAccept(t *testing.T) {
 		{[]string{"localhost:0"}, nil, "GET / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "GET / HTTP/1.0\r\n"},
 		{[]string{nonLoopBackIP + ":0"}, nil, "POST / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "POST / HTTP/1.0\r\n"},
 		{[]string{nonLoopBackIP + ":0"}, nil, "HEAD / HTTP/1.0\r\nhost: example.org\r\n\r\n", "200 OK\r\n", "HEAD / HTTP/1.0\r\n"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, "CONNECT \r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT \r\n"},
+		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, nil, "CONNECT / HTTP/1.0\r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT / HTTP/1.0\r\n"},
 		{[]string{"localhost:0"}, tlsConfig, "GET / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "GET / HTTP/1.0\r\n"},
 		{[]string{nonLoopBackIP + ":0"}, tlsConfig, "POST / HTTP/1.0\r\nHost: example.org\r\n\r\n", "200 OK\r\n", "POST / HTTP/1.0\r\n"},
 		{[]string{nonLoopBackIP + ":0"}, tlsConfig, "HEAD / HTTP/1.0\r\nhost: example.org\r\n\r\n", "200 OK\r\n", "HEAD / HTTP/1.0\r\n"},
-		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, "CONNECT \r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT \r\n"},
+		{[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}, tlsConfig, "CONNECT / HTTP/1.0\r\nHost: www.example.org\r\n\r\n", "200 OK\r\n", "CONNECT / HTTP/1.0\r\n"},
 	}
 
 	for i, testCase := range testCases {
@@ -643,11 +643,22 @@ func TestHTTPListenerAcceptError(t *testing.T) {
 				}
 			}()
 
-			_, err = bufio.NewReader(conn).ReadString('\n')
-			if err == nil {
-				t.Fatalf("Test %d: reply read: expected = EOF got = <nil>", i+1)
-			} else if err.Error() != "EOF" {
-				t.Fatalf("Test %d: reply read: expected = EOF got = %v", i+1, err)
+			if !testCase.secureClient && testCase.tlsConfig != nil {
+				buf := make([]byte, len(sslRequiredErrMsg))
+				var n int
+				n, err = io.ReadFull(conn, buf)
+				if err != nil {
+					t.Fatalf("Test %d: reply read: expected = <nil> got = %v", i+1, err)
+				} else if n != len(buf) {
+					t.Fatalf("Test %d: reply length: expected = %v got = %v", i+1, len(buf), n)
+				} else if !bytes.Equal(buf, sslRequiredErrMsg) {
+					t.Fatalf("Test %d: reply: expected = %v got = %v", i+1, string(sslRequiredErrMsg), string(buf))
+				}
+				continue
+			}
+
+			if _, err = bufio.NewReader(conn).ReadString('\n'); err != io.EOF {
+				t.Errorf("Test %d: reply read: expected = io.EOF, got = %s", i+1, err)
 			}
 
 			conn.Close()

cmd/http/server_test.go

@@ -17,12 +17,10 @@
 package http
 
 import (
-	"crypto/tls"
 	"fmt"
 	"net/http"
 	"reflect"
 	"testing"
-	"time"
 
 	"github.com/minio/minio/pkg/certs"
 )
@@ -96,78 +94,3 @@ func TestNewServer(t *testing.T) {
 		}
 	}
 }
-
-func TestServerTLSCiphers(t *testing.T) {
-	var unsupportedCipherSuites = []uint16{
-		tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA,    // Go stack contains (some) countermeasures against timing attacks (Lucky13)
-		tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
-		tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,    // Go stack contains (some) countermeasures against timing attacks (Lucky13)
-		tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA,        // Broken cipher
-		tls.TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA,     // Sweet32
-		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,      // Go stack contains (some) countermeasures against timing attacks (Lucky13)
-		tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256,   // No countermeasures against timing attacks
-		tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,      // Go stack contains (some) countermeasures against timing attacks (Lucky13)
-		tls.TLS_ECDHE_RSA_WITH_RC4_128_SHA,          // Broken cipher
-
-		// all RSA-PKCS1-v1.5 ciphers are disabled - danger of Bleichenbacher attack variants
-		tls.TLS_RSA_WITH_3DES_EDE_CBC_SHA,   // Sweet32
-		tls.TLS_RSA_WITH_AES_128_CBC_SHA,    // Go stack contains (some) countermeasures against timing attacks (Lucky13)
-		tls.TLS_RSA_WITH_AES_128_CBC_SHA256, // No countermeasures against timing attacks
-		tls.TLS_RSA_WITH_AES_256_CBC_SHA,    // Go stack contains (some) countermeasures against timing attacks (Lucky13)
-		tls.TLS_RSA_WITH_RC4_128_SHA,        // Broken cipher
-
-		tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure.
-		tls.TLS_RSA_WITH_AES_256_GCM_SHA384, // Disabled because of RSA-PKCS1-v1.5 - AES-GCM is considered secure.
-	}
-
-	testCases := []struct {
-		ciphers            []uint16
-		resetServerCiphers bool
-		expectErr          bool
-	}{
-		{nil, false, false},
-		{defaultCipherSuites, false, false},
-		{unsupportedCipherSuites, false, true},
-		{nil, true, false},
-	}
-
-	for i, testCase := range testCases {
-		func() {
-			addr := "127.0.0.1:" + getNextPort()
-
-			server := NewServer([]string{addr},
-				http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-					fmt.Fprintf(w, "Hello, world")
-				}), getCert)
-			if testCase.resetServerCiphers {
-				// Use Go default ciphers.
-				server.TLSConfig.CipherSuites = nil
-			}
-
-			go func() {
-				server.Start()
-			}()
-			defer server.Shutdown()
-
-			client := http.Client{
-				Transport: &http.Transport{
-					TLSClientConfig: &tls.Config{
-						InsecureSkipVerify: true,
-						CipherSuites:       testCase.ciphers,
-					},
-				},
-			}
-
-			// There is no guaranteed way to know whether the HTTP server is started successfully.
-			// The only option is to connect and check.  Hence below sleep is used as workaround.
-			time.Sleep(1 * time.Second)
-
-			_, err := client.Get("https://" + addr)
-			expectErr := (err != nil)
-
-			if expectErr != testCase.expectErr {
-				t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-			}
-		}()
-	}
-}

cmd/httprange.go

@@ -17,8 +17,8 @@
 package cmd
 
 import (
+	"errors"
 	"fmt"
-	"regexp"
 	"strconv"
 	"strings"
 )
@@ -27,27 +27,82 @@ const (
 	byteRangePrefix = "bytes="
 )
 
-// Valid byte position regexp
-var validBytePos = regexp.MustCompile(`^[0-9]+$`)
+// HTTPRangeSpec represents a range specification as supported by S3 GET
+// object request.
+//
+// Case 1: Not present -> represented by a nil RangeSpec
+// Case 2: bytes=1-10 (absolute start and end offsets) -> RangeSpec{false, 1, 10}
+// Case 3: bytes=10- (absolute start offset with end offset unspecified) -> RangeSpec{false, 10, -1}
+// Case 4: bytes=-30 (suffix length specification) -> RangeSpec{true, -30, -1}
+type HTTPRangeSpec struct {
+	// Does the range spec refer to a suffix of the object?
+	IsSuffixLength bool
 
-// HttpRange specifies the byte range to be sent to the client.
-type httpRange struct {
-	offsetBegin  int64
-	offsetEnd    int64
-	resourceSize int64
+	// Start and end offset specified in range spec
+	Start, End int64
 }
 
-// String populate range stringer interface
-func (hrange httpRange) String() string {
-	return fmt.Sprintf("bytes %d-%d/%d", hrange.offsetBegin, hrange.offsetEnd, hrange.resourceSize)
+// GetLength - get length of range
+func (h *HTTPRangeSpec) GetLength(resourceSize int64) (rangeLength int64, err error) {
+	switch {
+	case resourceSize < 0:
+		return 0, errors.New("Resource size cannot be negative")
+
+	case h == nil:
+		rangeLength = resourceSize
+
+	case h.IsSuffixLength:
+		specifiedLen := -h.Start
+		rangeLength = specifiedLen
+		if specifiedLen > resourceSize {
+			rangeLength = resourceSize
+		}
+
+	case h.Start >= resourceSize:
+		return 0, errInvalidRange
+
+	case h.End > -1:
+		end := h.End
+		if resourceSize <= end {
+			end = resourceSize - 1
+		}
+		rangeLength = end - h.Start + 1
+
+	case h.End == -1:
+		rangeLength = resourceSize - h.Start
+
+	default:
+		return 0, errors.New("Unexpected range specification case")
+	}
+
+	return rangeLength, nil
 }
 
-// getlength - get length from the range.
-func (hrange httpRange) getLength() int64 {
-	return 1 + hrange.offsetEnd - hrange.offsetBegin
+// GetOffsetLength computes the start offset and length of the range
+// given the size of the resource
+func (h *HTTPRangeSpec) GetOffsetLength(resourceSize int64) (start, length int64, err error) {
+	if h == nil {
+		// No range specified, implies whole object.
+		return 0, resourceSize, nil
+	}
+
+	length, err = h.GetLength(resourceSize)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	start = h.Start
+	if h.IsSuffixLength {
+		start = resourceSize + h.Start
+		if start < 0 {
+			start = 0
+		}
+	}
+	return start, length, nil
 }
 
-func parseRequestRange(rangeString string, resourceSize int64) (hrange *httpRange, err error) {
+// Parse a HTTP range header value into a HTTPRangeSpec
+func parseRequestRangeSpec(rangeString string) (hrange *HTTPRangeSpec, err error) {
 	// Return error if given range string doesn't start with byte range prefix.
 	if !strings.HasPrefix(rangeString, byteRangePrefix) {
 		return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix)
@@ -66,12 +121,12 @@ func parseRequestRange(rangeString string, resourceSize int64) (hrange *httpRang
 	offsetBegin := int64(-1)
 	// Convert offsetBeginString only if its not empty.
 	if len(offsetBeginString) > 0 {
-		if !validBytePos.MatchString(offsetBeginString) {
-			return nil, fmt.Errorf("'%s' does not have a valid first byte position value", rangeString)
-		}
-
-		if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil {
+		if offsetBeginString[0] == '+' {
+			return nil, fmt.Errorf("Byte position ('%s') must not have a sign", offsetBeginString)
+		} else if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil {
 			return nil, fmt.Errorf("'%s' does not have a valid first byte position value", rangeString)
+		} else if offsetBegin < 0 {
+			return nil, fmt.Errorf("First byte position is negative ('%d')", offsetBegin)
 		}
 	}
 
@@ -79,57 +134,30 @@ func parseRequestRange(rangeString string, resourceSize int64) (hrange *httpRang
 	offsetEnd := int64(-1)
 	// Convert offsetEndString only if its not empty.
 	if len(offsetEndString) > 0 {
-		if !validBytePos.MatchString(offsetEndString) {
-			return nil, fmt.Errorf("'%s' does not have a valid last byte position value", rangeString)
-		}
-
-		if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil {
+		if offsetEndString[0] == '+' {
+			return nil, fmt.Errorf("Byte position ('%s') must not have a sign", offsetEndString)
+		} else if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil {
 			return nil, fmt.Errorf("'%s' does not have a valid last byte position value", rangeString)
+		} else if offsetEnd < 0 {
+			return nil, fmt.Errorf("Last byte position is negative ('%d')", offsetEnd)
 		}
 	}
 
-	// rangeString contains first and last byte positions. eg. "bytes=2-5"
 	switch {
 	case offsetBegin > -1 && offsetEnd > -1:
 		if offsetBegin > offsetEnd {
-			// Last byte position is not greater than first byte position. eg. "bytes=5-2"
-			return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
-		}
-
-		// First and last byte positions should not be >= resourceSize.
-		if offsetBegin >= resourceSize {
 			return nil, errInvalidRange
 		}
-
-		if offsetEnd >= resourceSize {
-			offsetEnd = resourceSize - 1
-		}
+		return &HTTPRangeSpec{false, offsetBegin, offsetEnd}, nil
 	case offsetBegin > -1:
-		// rangeString contains only first byte position. eg. "bytes=8-"
-		if offsetBegin >= resourceSize {
-			// First byte position should not be >= resourceSize.
-			return nil, errInvalidRange
-		}
-
-		offsetEnd = resourceSize - 1
+		return &HTTPRangeSpec{false, offsetBegin, -1}, nil
 	case offsetEnd > -1:
-		// rangeString contains only last byte position. eg. "bytes=-3"
 		if offsetEnd == 0 {
-			// Last byte position should not be zero eg. "bytes=-0"
 			return nil, errInvalidRange
 		}
-
-		if offsetEnd >= resourceSize {
-			offsetBegin = 0
-		} else {
-			offsetBegin = resourceSize - offsetEnd
-		}
-
-		offsetEnd = resourceSize - 1
+		return &HTTPRangeSpec{true, -offsetEnd, -1}, nil
 	default:
 		// rangeString contains first and last byte positions missing. eg. "bytes=-"
 		return nil, fmt.Errorf("'%s' does not have valid range value", rangeString)
 	}
-
-	return &httpRange{offsetBegin, offsetEnd, resourceSize}, nil
 }

cmd/httprange_test.go

@@ -16,75 +16,87 @@
 
 package cmd
 
-import "testing"
+import (
+	"testing"
+)
 
-// Test parseRequestRange()
-func TestParseRequestRange(t *testing.T) {
-	// Test success cases.
-	successCases := []struct {
-		rangeString string
-		offsetBegin int64
-		offsetEnd   int64
-		length      int64
+func TestHTTPRequestRangeSpec(t *testing.T) {
+	resourceSize := int64(10)
+	validRangeSpecs := []struct {
+		spec                 string
+		expOffset, expLength int64
 	}{
-		{"bytes=2-5", 2, 5, 4},
-		{"bytes=2-20", 2, 9, 8},
-		{"bytes=2-2", 2, 2, 1},
-		{"bytes=0000-0006", 0, 6, 7},
-		{"bytes=2-", 2, 9, 8},
-		{"bytes=-4", 6, 9, 4},
-		{"bytes=-20", 0, 9, 10},
+		{"bytes=0-", 0, 10},
+		{"bytes=1-", 1, 9},
+		{"bytes=0-9", 0, 10},
+		{"bytes=1-10", 1, 9},
+		{"bytes=1-1", 1, 1},
+		{"bytes=2-5", 2, 4},
+		{"bytes=-5", 5, 5},
+		{"bytes=-1", 9, 1},
+		{"bytes=-1000", 0, 10},
 	}
-
-	for _, successCase := range successCases {
-		hrange, err := parseRequestRange(successCase.rangeString, 10)
+	for i, testCase := range validRangeSpecs {
+		rs, err := parseRequestRangeSpec(testCase.spec)
 		if err != nil {
-			t.Fatalf("expected: <nil>, got: %s", err)
+			t.Errorf("unexpected err: %v", err)
 		}
-
-		if hrange.offsetBegin != successCase.offsetBegin {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, hrange.offsetBegin)
+		o, l, err := rs.GetOffsetLength(resourceSize)
+		if err != nil {
+			t.Errorf("unexpected err: %v", err)
 		}
-
-		if hrange.offsetEnd != successCase.offsetEnd {
-			t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, hrange.offsetEnd)
-		}
-		if hrange.getLength() != successCase.length {
-			t.Fatalf("expected: %d, got: %d", successCase.length, hrange.getLength())
+		if o != testCase.expOffset || l != testCase.expLength {
+			t.Errorf("Case %d: got bad offset/length: %d,%d expected: %d,%d",
+				i, o, l, testCase.expOffset, testCase.expLength)
 		}
 	}
 
-	// Test invalid range strings.
-	invalidRangeStrings := []string{
-		"bytes=8",
-		"bytes=5-2",
-		"bytes=+2-5",
-		"bytes=2-+5",
-		"bytes=2--5",
+	unparsableRangeSpecs := []string{
 		"bytes=-",
+		"bytes==",
+		"bytes==1-10",
+		"bytes=",
+		"bytes=aa",
+		"aa",
 		"",
-		"2-5",
-		"bytes = 2-5",
-		"bytes=2 - 5",
-		"bytes=0-0,-1",
-		"bytes=2-5 ",
+		"bytes=1-10-",
+		"bytes=1--10",
+		"bytes=-1-10",
+		"bytes=0-+3",
+		"bytes=+3-+5",
+		"bytes=10-11,12-10", // Unsupported by S3/Minio (valid in RFC)
 	}
-	for _, rangeString := range invalidRangeStrings {
-		if _, err := parseRequestRange(rangeString, 10); err == nil {
-			t.Fatalf("expected: an error, got: <nil>")
+	for i, urs := range unparsableRangeSpecs {
+		rs, err := parseRequestRangeSpec(urs)
+		if err == nil {
+			t.Errorf("Case %d: Did not get an expected error - got %v", i, rs)
+		}
+		if err == errInvalidRange {
+			t.Errorf("Case %d: Got invalid range error instead of a parse error", i)
+		}
+		if rs != nil {
+			t.Errorf("Case %d: Got non-nil rs though err != nil: %v", i, rs)
 		}
 	}
 
-	// Test error range strings.
-	errorRangeString := []string{
+	invalidRangeSpecs := []string{
+		"bytes=5-3",
 		"bytes=10-10",
-		"bytes=20-30",
-		"bytes=20-",
+		"bytes=10-",
+		"bytes=100-",
 		"bytes=-0",
 	}
-	for _, rangeString := range errorRangeString {
-		if _, err := parseRequestRange(rangeString, 10); err != errInvalidRange {
-			t.Fatalf("expected: %s, got: %s", errInvalidRange, err)
+	for i, irs := range invalidRangeSpecs {
+		var err1, err2 error
+		var rs *HTTPRangeSpec
+		var o, l int64
+		rs, err1 = parseRequestRangeSpec(irs)
+		if err1 == nil {
+			o, l, err2 = rs.GetOffsetLength(resourceSize)
 		}
+		if err1 == errInvalidRange || (err1 == nil && err2 == errInvalidRange) {
+			continue
+		}
+		t.Errorf("Case %d: Expected errInvalidRange but: %v %v %d %d %v", i, rs, err1, o, l, err2)
 	}
 }

cmd/local-admin-client.go

@@ -19,7 +19,11 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"os"
+
+	"io/ioutil"
 )
 
 // localAdminClient - represents admin operation to be executed locally.
@@ -80,3 +84,40 @@ func (lc localAdminClient) GetConfig() ([]byte, error) {
 
 	return json.Marshal(globalServerConfig)
 }
+
+// StartProfiling - starts profiling on the local server.
+func (lc localAdminClient) StartProfiling(profiler string) error {
+	if globalProfiler != nil {
+		globalProfiler.Stop()
+	}
+	prof, err := startProfiler(profiler, "")
+	if err != nil {
+		return err
+	}
+	globalProfiler = prof
+	return nil
+}
+
+// DownloadProfilingData - stops and returns profiling data of the local server.
+func (lc localAdminClient) DownloadProfilingData() ([]byte, error) {
+	if globalProfiler == nil {
+		return nil, errors.New("profiler not enabled")
+	}
+
+	profilerPath := globalProfiler.Path()
+
+	// Stop the profiler
+	globalProfiler.Stop()
+
+	profilerFile, err := os.Open(profilerPath)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := ioutil.ReadAll(profilerFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}

cmd/lock-rpc-server.go

@@ -186,7 +186,7 @@ func NewLockRPCServer() (*xrpc.Server, error) {
 // Register distributed NS lock handlers.
 func registerDistNSLockRouter(router *mux.Router) {
 	rpcServer, err := NewLockRPCServer()
-	logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
+	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
 
 	// Start lock maintenance from all lock servers.
 	go startLockMaintenance(globalLockServer)

cmd/logger/reqinfo.go

@@ -117,6 +117,9 @@ func GetReqInfo(ctx context.Context) *ReqInfo {
 		if ok {
 			return r
 		}
+		r = &ReqInfo{}
+		SetReqInfo(ctx, r)
+		return r
 	}
 	return nil
 }

cmd/namespace-lock.go

@@ -86,9 +86,9 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
 			locker = &(receiver.ll)
 		} else {
 			host, err := xnet.ParseHost(endpoint.Host)
-			logger.FatalIf(err, "Unable to parse Lock RPC Host", context.Background())
+			logger.FatalIf(err, "Unable to parse Lock RPC Host")
 			locker, err = NewLockRPCClient(host)
-			logger.FatalIf(err, "Unable to initialize Lock RPC Client", context.Background())
+			logger.FatalIf(err, "Unable to initialize Lock RPC Client")
 		}
 
 		clnts = append(clnts, locker)

cmd/object-api-common.go

@@ -100,7 +100,7 @@ func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) {
 		return newPosix(endpoint.Path)
 	}
 
-	return newStorageRPC(endpoint), nil
+	return newStorageRESTClient(endpoint), nil
 }
 
 // Cleanup a directory recursively.

cmd/object-api-datatypes.go

@@ -258,6 +258,9 @@ type PartInfo struct {
 
 	// Size in bytes of the part.
 	Size int64
+
+	// Decompressed Size.
+	ActualSize int64
 }
 
 // MultipartInfo - represents metadata in progress multipart upload.

cmd/object-api-errors.go

@@ -395,3 +395,9 @@ func isInsufficientReadQuorum(err error) bool {
 	_, ok := err.(InsufficientReadQuorum)
 	return ok
 }
+
+// isInsufficientWriteQuorum - Check if error type is InsufficientWriteQuorum.
+func isInsufficientWriteQuorum(err error) bool {
+	_, ok := err.(InsufficientWriteQuorum)
+	return ok
+}

cmd/object-api-interface.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"io"
+	"net/http"
 
 	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio/pkg/hash"
@@ -31,6 +32,15 @@ type ObjectOptions struct {
 	ServerSideEncryption encrypt.ServerSide
 }
 
+// LockType represents required locking for ObjectLayer operations
+type LockType int
+
+const (
+	noLock LockType = iota
+	readLock
+	writeLock
+)
+
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
 	// Storage operations.
@@ -47,6 +57,13 @@ type ObjectLayer interface {
 
 	// Object operations.
 
+	// GetObjectNInfo returns a GetObjectReader that satisfies the
+	// ReadCloser interface. The Close method unlocks the object
+	// after reading, so it must always be called after usage.
+	//
+	// IMPORTANTLY, when implementations return err != nil, this
+	// function MUST NOT return a non-nil ReadCloser.
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error)
 	GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -79,4 +96,7 @@ type ObjectLayer interface {
 	// Supported operations check
 	IsNotificationSupported() bool
 	IsEncryptionSupported() bool
+
+	// Compression support check.
+	IsCompressionSupported() bool
 }

cmd/object-api-utils.go

@@ -20,15 +20,23 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
+	"io"
 	"math/rand"
+	"net/http"
 	"path"
 	"runtime"
+	"strconv"
 	"strings"
+	"sync"
 	"time"
 	"unicode/utf8"
 
+	snappy "github.com/golang/snappy"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dns"
+	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/minio/pkg/wildcard"
 	"github.com/skyrings/skyring-common/tools/uuid"
 )
 
@@ -296,9 +304,296 @@ func getRandomHostPort(records []dns.SrvRecord) (string, int) {
 	return srvRecord.Host, srvRecord.Port
 }
 
+// IsCompressed returns true if the object is marked as compressed.
+func (o ObjectInfo) IsCompressed() bool {
+	_, ok := o.UserDefined[ReservedMetadataPrefix+"compression"]
+	return ok
+}
+
+// GetActualSize - read the decompressed size from the meta json.
+func (o ObjectInfo) GetActualSize() int64 {
+	metadata := o.UserDefined
+	sizeStr, ok := metadata[ReservedMetadataPrefix+"actual-size"]
+	if ok {
+		size, err := strconv.ParseInt(sizeStr, 10, 64)
+		if err == nil {
+			return size
+		}
+	}
+	return -1
+}
+
+// Disabling compression for encrypted enabled requests.
+// Using compression and encryption together enables room for side channel attacks.
+// Eliminate non-compressible objects by extensions/content-types.
+func isCompressible(header http.Header, object string) bool {
+	if hasServerSideEncryptionHeader(header) || excludeForCompression(header, object) {
+		return false
+	}
+	return true
+}
+
+// Eliminate the non-compressible objects.
+func excludeForCompression(header http.Header, object string) bool {
+	objStr := object
+	contentType := header.Get("Content-Type")
+	if globalIsCompressionEnabled {
+		// We strictly disable compression for standard extensions/content-types (`compressed`).
+		if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) {
+			return true
+		}
+		// Filter compression includes.
+		if len(globalCompressExtensions) > 0 || len(globalCompressMimeTypes) > 0 {
+			extensions := globalCompressExtensions
+			mimeTypes := globalCompressMimeTypes
+			if hasStringSuffixInSlice(objStr, extensions) || hasPattern(mimeTypes, contentType) {
+				return false
+			}
+			return true
+		}
+		return false
+	}
+	return true
+}
+
+// Utility which returns if a string is present in the list.
+func hasStringSuffixInSlice(str string, list []string) bool {
+	for _, v := range list {
+		if strings.HasSuffix(str, v) {
+			return true
+		}
+	}
+	return false
+}
+
+// Returns true if any of the given wildcard patterns match the matchStr.
+func hasPattern(patterns []string, matchStr string) bool {
+	for _, pattern := range patterns {
+		if ok := wildcard.MatchSimple(pattern, matchStr); ok {
+			return true
+		}
+	}
+	return false
+}
+
+// Returns the part file name which matches the partNumber and etag.
+func getPartFile(entries []string, partNumber int, etag string) string {
+	for _, entry := range entries {
+		if strings.HasPrefix(entry, fmt.Sprintf("%.5d.%s.", partNumber, etag)) {
+			return entry
+		}
+	}
+	return ""
+}
+
+// Returs the compressed offset which should be skipped.
+func getCompressedOffsets(objectInfo ObjectInfo, offset int64) (int64, int64) {
+	var compressedOffset int64
+	var skipLength int64
+	var cumulativeActualSize int64
+	if len(objectInfo.Parts) > 0 {
+		for _, part := range objectInfo.Parts {
+			cumulativeActualSize += part.ActualSize
+			if cumulativeActualSize <= offset {
+				compressedOffset += part.Size
+			} else {
+				skipLength = cumulativeActualSize - part.ActualSize
+				break
+			}
+		}
+	}
+	return compressedOffset, offset - skipLength
+}
+
 // byBucketName is a collection satisfying sort.Interface.
 type byBucketName []BucketInfo
 
 func (d byBucketName) Len() int           { return len(d) }
 func (d byBucketName) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 func (d byBucketName) Less(i, j int) bool { return d[i].Name < d[j].Name }
+
+// GetObjectReader is a type that wraps a reader with a lock to
+// provide a ReadCloser interface that unlocks on Close()
+type GetObjectReader struct {
+	ObjInfo ObjectInfo
+	pReader io.Reader
+
+	cleanUpFns []func()
+	once       sync.Once
+}
+
+// NewGetObjectReaderFromReader sets up a GetObjectReader with a given
+// reader. This ignores any object properties.
+func NewGetObjectReaderFromReader(r io.Reader, oi ObjectInfo, cleanupFns ...func()) *GetObjectReader {
+	return &GetObjectReader{
+		ObjInfo:    oi,
+		pReader:    r,
+		cleanUpFns: cleanupFns,
+	}
+}
+
+// ObjReaderFn is a function type that takes a reader and returns
+// GetObjectReader and an error. Request headers are passed to provide
+// encryption parameters. cleanupFns allow cleanup funcs to be
+// registered for calling after usage of the reader.
+type ObjReaderFn func(inputReader io.Reader, h http.Header, cleanupFns ...func()) (r *GetObjectReader, err error)
+
+// NewGetObjectReader creates a new GetObjectReader. The cleanUpFns
+// are called on Close() in reverse order as passed here. NOTE: It is
+// assumed that clean up functions do not panic (otherwise, they may
+// not all run!).
+func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func()) (
+	fn ObjReaderFn, off, length int64, err error) {
+
+	// Call the clean-up functions immediately in case of exit
+	// with error
+	defer func() {
+		if err != nil {
+			for i := len(cleanUpFns) - 1; i >= 0; i-- {
+				cleanUpFns[i]()
+			}
+		}
+	}()
+
+	isEncrypted := crypto.IsEncrypted(oi.UserDefined)
+	isCompressed := oi.IsCompressed()
+	var skipLen int64
+	// Calculate range to read (different for
+	// e.g. encrypted/compressed objects)
+	switch {
+	case isEncrypted:
+		var seqNumber uint32
+		var partStart int
+		off, length, skipLen, seqNumber, partStart, err = oi.GetDecryptedRange(rs)
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		var decSize int64
+		decSize, err = oi.DecryptedSize()
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		var decRangeLength int64
+		decRangeLength, err = rs.GetLength(decSize)
+		if err != nil {
+			return nil, 0, 0, err
+		}
+
+		// We define a closure that performs decryption given
+		// a reader that returns the desired range of
+		// encrypted bytes. The header parameter is used to
+		// provide encryption parameters.
+		fn = func(inputReader io.Reader, h http.Header, cFns ...func()) (r *GetObjectReader, err error) {
+
+			copySource := h.Get(crypto.SSECopyAlgorithm) != ""
+
+			cFns = append(cleanUpFns, cFns...)
+			// Attach decrypter on inputReader
+			var decReader io.Reader
+			decReader, err = DecryptBlocksRequestR(inputReader, h,
+				off, length, seqNumber, partStart, oi, copySource)
+			if err != nil {
+				// Call the cleanup funcs
+				for i := len(cFns) - 1; i >= 0; i-- {
+					cFns[i]()
+				}
+				return nil, err
+			}
+
+			// Apply the skipLen and limit on the
+			// decrypted stream
+			decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength)
+
+			// Assemble the GetObjectReader
+			r = &GetObjectReader{
+				ObjInfo:    oi,
+				pReader:    decReader,
+				cleanUpFns: cFns,
+			}
+			return r, nil
+		}
+	case isCompressed:
+		// Read the decompressed size from the meta.json.
+		actualSize := oi.GetActualSize()
+		if actualSize < 0 {
+			return nil, 0, 0, errInvalidDecompressedSize
+		}
+		off, length = int64(0), oi.Size
+		decOff, decLength := int64(0), actualSize
+		if rs != nil {
+			off, length, err = rs.GetOffsetLength(actualSize)
+			if err != nil {
+				return nil, 0, 0, err
+			}
+			// Incase of range based queries on multiparts, the offset and length are reduced.
+			off, decOff = getCompressedOffsets(oi, off)
+			decLength = length
+			length = oi.Size - off
+
+			// For negative length we read everything.
+			if decLength < 0 {
+				decLength = actualSize - decOff
+			}
+
+			// Reply back invalid range if the input offset and length fall out of range.
+			if decOff > actualSize || decOff+decLength > actualSize {
+				return nil, 0, 0, errInvalidRange
+			}
+		}
+		fn = func(inputReader io.Reader, _ http.Header, cFns ...func()) (r *GetObjectReader, err error) {
+			// Decompression reader.
+			snappyReader := snappy.NewReader(inputReader)
+			// Apply the skipLen and limit on the
+			// decompressed stream
+			decReader := io.LimitReader(ioutil.NewSkipReader(snappyReader, decOff), decLength)
+			oi.Size = decLength
+
+			// Assemble the GetObjectReader
+			r = &GetObjectReader{
+				ObjInfo:    oi,
+				pReader:    decReader,
+				cleanUpFns: append(cleanUpFns, cFns...),
+			}
+			return r, nil
+		}
+
+	default:
+		off, length, err = rs.GetOffsetLength(oi.Size)
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		fn = func(inputReader io.Reader, _ http.Header, cFns ...func()) (r *GetObjectReader, err error) {
+			r = &GetObjectReader{
+				ObjInfo:    oi,
+				pReader:    inputReader,
+				cleanUpFns: append(cleanUpFns, cFns...),
+			}
+			return r, nil
+		}
+	}
+
+	return fn, off, length, nil
+}
+
+// Close - calls the cleanup actions in reverse order
+func (g *GetObjectReader) Close() error {
+	// sync.Once is used here to ensure that Close() is
+	// idempotent.
+	g.once.Do(func() {
+		for i := len(g.cleanUpFns) - 1; i >= 0; i-- {
+			g.cleanUpFns[i]()
+		}
+	})
+	return nil
+}
+
+// Read - to implement Reader interface.
+func (g *GetObjectReader) Read(p []byte) (n int, err error) {
+	n, err = g.pReader.Read(p)
+	if err != nil {
+		// Calling code may not Close() in case of error, so
+		// we ensure it.
+		g.Close()
+	}
+	return
+}

cmd/object-api-utils_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"net/http"
 	"reflect"
 	"testing"
 )
@@ -296,3 +297,250 @@ func TestCleanMetadataKeys(t *testing.T) {
 		}
 	}
 }
+
+// Tests isCompressed method
+func TestIsCompressed(t *testing.T) {
+	testCases := []struct {
+		objInfo ObjectInfo
+		result  bool
+	}{
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+					"content-type": "application/octet-stream",
+					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+			},
+			result: true,
+		},
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-XYZ": "golang/snappy/LZ77",
+					"content-type": "application/octet-stream",
+					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+			},
+			result: false,
+		},
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"content-type": "application/octet-stream",
+					"etag": "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+			},
+			result: false,
+		},
+	}
+	for i, test := range testCases {
+		got := test.objInfo.IsCompressed()
+		if got != test.result {
+			t.Errorf("Test %d - expected %v but received %v",
+				i+1, test.result, got)
+		}
+	}
+}
+
+// Tests excludeForCompression.
+func TestExcludeForCompression(t *testing.T) {
+	testCases := []struct {
+		object string
+		header http.Header
+		result bool
+	}{
+		{
+			object: "object.txt",
+			header: http.Header{
+				"Content-Type": []string{"application/zip"},
+			},
+			result: true,
+		},
+		{
+			object: "object.zip",
+			header: http.Header{
+				"Content-Type": []string{"application/XYZ"},
+			},
+			result: true,
+		},
+		{
+			object: "object.json",
+			header: http.Header{
+				"Content-Type": []string{"application/json"},
+			},
+			result: false,
+		},
+		{
+			object: "object.txt",
+			header: http.Header{
+				"Content-Type": []string{"text/plain"},
+			},
+			result: false,
+		},
+	}
+	for i, test := range testCases {
+		globalIsCompressionEnabled = true
+		got := excludeForCompression(test.header, test.object)
+		globalIsCompressionEnabled = false
+		if got != test.result {
+			t.Errorf("Test %d - expected %v but received %v",
+				i+1, test.result, got)
+		}
+	}
+}
+
+// Test getPartFile function.
+func TestGetPartFile(t *testing.T) {
+	testCases := []struct {
+		entries    []string
+		partNumber int
+		etag       string
+		result     string
+	}{
+		{
+			entries:    []string{"00001.8a034f82cb9cb31140d87d3ce2a9ede3.67108864", "fs.json", "00002.d73d8ab724016dfb051e2d3584495c54.32891137"},
+			partNumber: 1,
+			etag:       "8a034f82cb9cb31140d87d3ce2a9ede3",
+			result:     "00001.8a034f82cb9cb31140d87d3ce2a9ede3.67108864",
+		},
+		{
+			entries:    []string{"00001.8a034f82cb9cb31140d87d3ce2a9ede3.67108864", "fs.json", "00002.d73d8ab724016dfb051e2d3584495c54.32891137"},
+			partNumber: 2,
+			etag:       "d73d8ab724016dfb051e2d3584495c54",
+			result:     "00002.d73d8ab724016dfb051e2d3584495c54.32891137",
+		},
+		{
+			entries:    []string{"00001.8a034f82cb9cb31140d87d3ce2a9ede3.67108864", "fs.json", "00002.d73d8ab724016dfb051e2d3584495c54.32891137"},
+			partNumber: 1,
+			etag:       "d73d8ab724016dfb051e2d3584495c54",
+			result:     "",
+		},
+	}
+	for i, test := range testCases {
+		got := getPartFile(test.entries, test.partNumber, test.etag)
+		if got != test.result {
+			t.Errorf("Test %d - expected %s but received %s",
+				i+1, test.result, got)
+		}
+	}
+}
+
+func TestGetActualSize(t *testing.T) {
+	testCases := []struct {
+		objInfo ObjectInfo
+		result  int64
+	}{
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+					"X-Minio-Internal-actual-size": "100000001",
+					"content-type":                 "application/octet-stream",
+					"etag":                         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+				Parts: []objectPartInfo{
+					{
+						Size:       39235668,
+						ActualSize: 67108864,
+					},
+					{
+						Size:       19177372,
+						ActualSize: 32891137,
+					},
+				},
+			},
+			result: 100000001,
+		},
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+					"X-Minio-Internal-actual-size": "841",
+					"content-type":                 "application/octet-stream",
+					"etag":                         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+				Parts: []objectPartInfo{},
+			},
+			result: 841,
+		},
+		{
+			objInfo: ObjectInfo{
+				UserDefined: map[string]string{"X-Minio-Internal-compression": "golang/snappy/LZ77",
+					"content-type": "application/octet-stream",
+					"etag":         "b3ff3ef3789147152fbfbc50efba4bfd-2"},
+				Parts: []objectPartInfo{},
+			},
+			result: -1,
+		},
+	}
+	for i, test := range testCases {
+		got := test.objInfo.GetActualSize()
+		if got != test.result {
+			t.Errorf("Test %d - expected %d but received %d",
+				i+1, test.result, got)
+		}
+	}
+}
+
+func TestGetCompressedOffsets(t *testing.T) {
+	testCases := []struct {
+		objInfo           ObjectInfo
+		offset            int64
+		startOffset       int64
+		snappyStartOffset int64
+	}{
+		{
+			objInfo: ObjectInfo{
+				Parts: []objectPartInfo{
+					{
+						Size:       39235668,
+						ActualSize: 67108864,
+					},
+					{
+						Size:       19177372,
+						ActualSize: 32891137,
+					},
+				},
+			},
+			offset:            79109865,
+			startOffset:       39235668,
+			snappyStartOffset: 12001001,
+		},
+		{
+			objInfo: ObjectInfo{
+				Parts: []objectPartInfo{
+					{
+						Size:       39235668,
+						ActualSize: 67108864,
+					},
+					{
+						Size:       19177372,
+						ActualSize: 32891137,
+					},
+				},
+			},
+			offset:            19109865,
+			startOffset:       0,
+			snappyStartOffset: 19109865,
+		},
+		{
+			objInfo: ObjectInfo{
+				Parts: []objectPartInfo{
+					{
+						Size:       39235668,
+						ActualSize: 67108864,
+					},
+					{
+						Size:       19177372,
+						ActualSize: 32891137,
+					},
+				},
+			},
+			offset:            0,
+			startOffset:       0,
+			snappyStartOffset: 0,
+		},
+	}
+	for i, test := range testCases {
+		startOffset, snappyStartOffset := getCompressedOffsets(test.objInfo, test.offset)
+		if startOffset != test.startOffset {
+			t.Errorf("Test %d - expected startOffset %d but received %d",
+				i+1, test.startOffset, startOffset)
+		}
+		if snappyStartOffset != test.snappyStartOffset {
+			t.Errorf("Test %d - expected snappyOffset %d but received %d",
+				i+1, test.snappyStartOffset, snappyStartOffset)
+		}
+	}
+}

cmd/object-handlers_test.go

@@ -19,9 +19,13 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
+	"encoding/base64"
 	"encoding/xml"
 	"fmt"
 	"io"
+	"strings"
+
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -31,7 +35,9 @@ import (
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/auth"
+	ioutilx "github.com/minio/minio/pkg/ioutil"
 )
 
 // Type to capture different modifications to API request to simulate failure cases.
@@ -129,7 +135,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get Object end point.
 		req, err := newTestSignedRequestV4("HEAD", getHeadObjectURL("", testCase.bucketName, testCase.objectName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: <ERROR> %v", i+1, instanceType, err)
 		}
@@ -147,7 +153,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for Head Object endpoint.
 		reqV2, err := newTestSignedRequestV2("HEAD", getHeadObjectURL("", testCase.bucketName, testCase.objectName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: <ERROR> %v", i+1, instanceType, err)
@@ -182,7 +188,7 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	nilBucket := "dummy-bucket"
 	nilObject := "dummy-object"
 	nilReq, err := newTestSignedRequestV4("HEAD", getGetObjectURL("", nilBucket, nilObject),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -192,14 +198,139 @@ func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
 
+func TestAPIHeadObjectHandlerWithEncryption(t *testing.T) {
+	globalPolicySys = NewPolicySys()
+	defer func() { globalPolicySys = nil }()
+
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(t, testAPIHeadObjectHandlerWithEncryption, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject", "HeadObject"})
+}
+
+func testAPIHeadObjectHandlerWithEncryption(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials auth.Credentials, t *testing.T) {
+
+	// Set SSL to on to do encryption tests
+	globalIsSSL = true
+	defer func() { globalIsSSL = false }()
+
+	var (
+		oneMiB        int64 = 1024 * 1024
+		key32Bytes          = generateBytesData(32 * humanize.Byte)
+		key32BytesMd5       = md5.Sum(key32Bytes)
+		metaWithSSEC        = map[string]string{
+			crypto.SSECAlgorithm: crypto.SSEAlgorithmAES256,
+			crypto.SSECKey:       base64.StdEncoding.EncodeToString(key32Bytes),
+			crypto.SSECKeyMD5:    base64.StdEncoding.EncodeToString(key32BytesMd5[:]),
+		}
+		mapCopy = func(m map[string]string) map[string]string {
+			r := make(map[string]string, len(m))
+			for k, v := range m {
+				r[k] = v
+			}
+			return r
+		}
+	)
+
+	type ObjectInput struct {
+		objectName  string
+		partLengths []int64
+
+		metaData map[string]string
+	}
+
+	objectLength := func(oi ObjectInput) (sum int64) {
+		for _, l := range oi.partLengths {
+			sum += l
+		}
+		return
+	}
+
+	// set of inputs for uploading the objects before tests for
+	// downloading is done. Data bytes are from DummyDataGen.
+	objectInputs := []ObjectInput{
+		// Unencrypted objects
+		{"nothing", []int64{0}, nil},
+		{"small-1", []int64{509}, nil},
+
+		{"mp-1", []int64{5 * oneMiB, 1}, nil},
+		{"mp-2", []int64{5487701, 5487799, 3}, nil},
+
+		// Encrypted object
+		{"enc-nothing", []int64{0}, mapCopy(metaWithSSEC)},
+		{"enc-small-1", []int64{509}, mapCopy(metaWithSSEC)},
+
+		{"enc-mp-1", []int64{5 * oneMiB, 1}, mapCopy(metaWithSSEC)},
+		{"enc-mp-2", []int64{5487701, 5487799, 3}, mapCopy(metaWithSSEC)},
+	}
+
+	// iterate through the above set of inputs and upload the object.
+	for _, input := range objectInputs {
+		uploadTestObject(t, apiRouter, credentials, bucketName, input.objectName, input.partLengths, input.metaData, false)
+	}
+
+	for i, input := range objectInputs {
+		// initialize HTTP NewRecorder, this records any
+		// mutations to response writer inside the handler.
+		rec := httptest.NewRecorder()
+		// construct HTTP request for HEAD object.
+		req, err := newTestSignedRequestV4("HEAD", getHeadObjectURL("", bucketName, input.objectName),
+			0, nil, credentials.AccessKey, credentials.SecretKey, nil)
+		if err != nil {
+			t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: <ERROR> %v", i+1, instanceType, err)
+		}
+		// Since `apiRouter` satisfies `http.Handler` it has a
+		// ServeHTTP to execute the logic of the handler.
+		apiRouter.ServeHTTP(rec, req)
+
+		isEnc := false
+		expected := 200
+		if strings.HasPrefix(input.objectName, "enc-") {
+			isEnc = true
+			expected = 400
+		}
+		if rec.Code != expected {
+			t.Errorf("Test %d: expected code %d but got %d for object %s", i+1, expected, rec.Code, input.objectName)
+		}
+
+		contentLength := rec.Header().Get("Content-Length")
+		if isEnc {
+			// initialize HTTP NewRecorder, this records any
+			// mutations to response writer inside the handler.
+			rec := httptest.NewRecorder()
+			// construct HTTP request for HEAD object.
+			req, err := newTestSignedRequestV4("HEAD", getHeadObjectURL("", bucketName, input.objectName),
+				0, nil, credentials.AccessKey, credentials.SecretKey, input.metaData)
+			if err != nil {
+				t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: <ERROR> %v", i+1, instanceType, err)
+			}
+			// Since `apiRouter` satisfies `http.Handler` it has a
+			// ServeHTTP to execute the logic of the handler.
+			apiRouter.ServeHTTP(rec, req)
+
+			if rec.Code != 200 {
+				t.Errorf("Test %d: Did not receive a 200 response: %d", i+1, rec.Code)
+			}
+			contentLength = rec.Header().Get("Content-Length")
+		}
+
+		if contentLength != fmt.Sprintf("%d", objectLength(input)) {
+			t.Errorf("Test %d: Content length is mismatching: got %s (expected: %d)", i+1, contentLength, objectLength(input))
+		}
+	}
+}
+
 // Wrapper for calling GetObject API handler tests for both XL multiple disks and FS single drive setup.
 func TestAPIGetObjectHandler(t *testing.T) {
+	globalPolicySys = NewPolicySys()
+	defer func() { globalPolicySys = nil }()
+
 	defer DetectTestLeak(t)()
 	ExecObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"})
 }
 
 func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
+
 	objectName := "test-object"
 	// set of byte data for PutObject.
 	// object has to be created before running tests for GetObject.
@@ -376,7 +507,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get Object end point.
 		req, err := newTestSignedRequestV4("GET", getGetObjectURL("", testCase.bucketName, testCase.objectName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for Get Object: <ERROR> %v", i+1, err)
@@ -406,7 +537,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for  GET Object endpoint.
 		reqV2, err := newTestSignedRequestV2("GET", getGetObjectURL("", testCase.bucketName, testCase.objectName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for GetObject: <ERROR> %v", i+1, instanceType, err)
@@ -455,7 +586,7 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	nilBucket := "dummy-bucket"
 	nilObject := "dummy-object"
 	nilReq, err := newTestSignedRequestV4("GET", getGetObjectURL("", nilBucket, nilObject),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -465,6 +596,204 @@ func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
 
+// Wrapper for calling GetObject API handler tests for both XL multiple disks and FS single drive setup.
+func TestAPIGetObjectWithMPHandler(t *testing.T) {
+	globalPolicySys = NewPolicySys()
+	defer func() { globalPolicySys = nil }()
+
+	defer DetectTestLeak(t)()
+	ExecObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"})
+}
+
+func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
+	credentials auth.Credentials, t *testing.T) {
+
+	// Set SSL to on to do encryption tests
+	globalIsSSL = true
+	defer func() { globalIsSSL = false }()
+
+	var (
+		oneMiB        int64 = 1024 * 1024
+		key32Bytes          = generateBytesData(32 * humanize.Byte)
+		key32BytesMd5       = md5.Sum(key32Bytes)
+		metaWithSSEC        = map[string]string{
+			crypto.SSECAlgorithm: crypto.SSEAlgorithmAES256,
+			crypto.SSECKey:       base64.StdEncoding.EncodeToString(key32Bytes),
+			crypto.SSECKeyMD5:    base64.StdEncoding.EncodeToString(key32BytesMd5[:]),
+		}
+		mapCopy = func(m map[string]string) map[string]string {
+			r := make(map[string]string, len(m))
+			for k, v := range m {
+				r[k] = v
+			}
+			return r
+		}
+	)
+
+	type ObjectInput struct {
+		objectName  string
+		partLengths []int64
+
+		metaData map[string]string
+	}
+
+	objectLength := func(oi ObjectInput) (sum int64) {
+		for _, l := range oi.partLengths {
+			sum += l
+		}
+		return
+	}
+
+	// set of inputs for uploading the objects before tests for
+	// downloading is done. Data bytes are from DummyDataGen.
+	objectInputs := []ObjectInput{
+		// // cases 0-3: small single part objects
+		{"nothing", []int64{0}, make(map[string]string)},
+		{"small-0", []int64{11}, make(map[string]string)},
+		{"small-1", []int64{509}, make(map[string]string)},
+		{"small-2", []int64{5 * oneMiB}, make(map[string]string)},
+		// // // cases 4-7: multipart part objects
+		{"mp-0", []int64{5 * oneMiB, 1}, make(map[string]string)},
+		{"mp-1", []int64{5*oneMiB + 1, 1}, make(map[string]string)},
+		{"mp-2", []int64{5487701, 5487799, 3}, make(map[string]string)},
+		{"mp-3", []int64{10499807, 10499963, 7}, make(map[string]string)},
+		// cases 8-11: small single part objects with encryption
+		{"enc-nothing", []int64{0}, mapCopy(metaWithSSEC)},
+		{"enc-small-0", []int64{11}, mapCopy(metaWithSSEC)},
+		{"enc-small-1", []int64{509}, mapCopy(metaWithSSEC)},
+		{"enc-small-2", []int64{5 * oneMiB}, mapCopy(metaWithSSEC)},
+		// cases 12-15: multipart part objects with encryption
+		{"enc-mp-0", []int64{5 * oneMiB, 1}, mapCopy(metaWithSSEC)},
+		{"enc-mp-1", []int64{5*oneMiB + 1, 1}, mapCopy(metaWithSSEC)},
+		{"enc-mp-2", []int64{5487701, 5487799, 3}, mapCopy(metaWithSSEC)},
+		{"enc-mp-3", []int64{10499807, 10499963, 7}, mapCopy(metaWithSSEC)},
+	}
+
+	// iterate through the above set of inputs and upload the object.
+	for _, input := range objectInputs {
+		uploadTestObject(t, apiRouter, credentials, bucketName, input.objectName, input.partLengths, input.metaData, false)
+	}
+
+	// function type for creating signed requests - used to repeat
+	// requests with V2 and V4 signing.
+	type testSignedReqFn func(method, urlStr string, contentLength int64,
+		body io.ReadSeeker, accessKey, secretKey string, metamap map[string]string) (*http.Request,
+		error)
+
+	mkGetReq := func(oi ObjectInput, byteRange string, i int, mkSignedReq testSignedReqFn) {
+		object := oi.objectName
+		rec := httptest.NewRecorder()
+		req, err := mkSignedReq("GET", getGetObjectURL("", bucketName, object),
+			0, nil, credentials.AccessKey, credentials.SecretKey, oi.metaData)
+		if err != nil {
+			t.Fatalf("Object: %s Case %d ByteRange: %s: Failed to create HTTP request for Get Object: <ERROR> %v",
+				object, i+1, byteRange, err)
+		}
+
+		if byteRange != "" {
+			req.Header.Add("Range", byteRange)
+		}
+
+		apiRouter.ServeHTTP(rec, req)
+
+		// Check response code (we make only valid requests in
+		// this test)
+		if rec.Code != http.StatusPartialContent && rec.Code != http.StatusOK {
+			bd, err1 := ioutil.ReadAll(rec.Body)
+			t.Fatalf("%s Object: %s Case %d ByteRange: %s: Got response status `%d` and body: %s,%v",
+				instanceType, object, i+1, byteRange, rec.Code, string(bd), err1)
+		}
+
+		var off, length int64
+		var rs *HTTPRangeSpec
+		if byteRange != "" {
+			rs, err = parseRequestRangeSpec(byteRange)
+			if err != nil {
+				t.Fatalf("Object: %s Case %d ByteRange: %s: Unexpected err: %v", object, i+1, byteRange, err)
+			}
+		}
+		off, length, err = rs.GetOffsetLength(objectLength(oi))
+		if err != nil {
+			t.Fatalf("Object: %s Case %d ByteRange: %s: Unexpected err: %v", object, i+1, byteRange, err)
+		}
+
+		readers := []io.Reader{}
+		cumulativeSum := int64(0)
+		for _, p := range oi.partLengths {
+			readers = append(readers, NewDummyDataGen(p, cumulativeSum))
+			cumulativeSum += p
+		}
+		refReader := io.LimitReader(ioutilx.NewSkipReader(io.MultiReader(readers...), off), length)
+		if ok, msg := cmpReaders(refReader, rec.Body); !ok {
+			t.Fatalf("(%s) Object: %s Case %d ByteRange: %s --> data mismatch! (msg: %s)", instanceType, oi.objectName, i+1, byteRange, msg)
+		}
+	}
+
+	// Iterate over each uploaded object and do a bunch of get
+	// requests on them.
+	caseNumber := 0
+	signFns := []testSignedReqFn{newTestSignedRequestV2, newTestSignedRequestV4}
+	for _, oi := range objectInputs {
+		objLen := objectLength(oi)
+		for _, sf := range signFns {
+			// Read whole object
+			mkGetReq(oi, "", caseNumber, sf)
+			caseNumber++
+
+			// No range requests are possible if the
+			// object length is 0
+			if objLen == 0 {
+				continue
+			}
+
+			// Various ranges to query - all are valid!
+			rangeHdrs := []string{
+				// Read first byte of object
+				fmt.Sprintf("bytes=%d-%d", 0, 0),
+				// Read second byte of object
+				fmt.Sprintf("bytes=%d-%d", 1, 1),
+				// Read last byte of object
+				fmt.Sprintf("bytes=-%d", 1),
+				// Read all but first byte of object
+				"bytes=1-",
+				// Read first half of object
+				fmt.Sprintf("bytes=%d-%d", 0, objLen/2),
+				// Read last half of object
+				fmt.Sprintf("bytes=-%d", objLen/2),
+				// Read middle half of object
+				fmt.Sprintf("bytes=%d-%d", objLen/4, objLen*3/4),
+				// Read 100MiB of the object from the beginning
+				fmt.Sprintf("bytes=%d-%d", 0, 100*humanize.MiByte),
+				// Read 100MiB of the object from the end
+				fmt.Sprintf("bytes=-%d", 100*humanize.MiByte),
+			}
+			for _, rangeHdr := range rangeHdrs {
+				mkGetReq(oi, rangeHdr, caseNumber, sf)
+				caseNumber++
+			}
+		}
+
+	}
+
+	// HTTP request for testing when `objectLayer` is set to `nil`.
+	// There is no need to use an existing bucket and valid input for creating the request
+	// since the `objectLayer==nil`  check is performed before any other checks inside the handlers.
+	// The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called.
+
+	nilBucket := "dummy-bucket"
+	nilObject := "dummy-object"
+	nilReq, err := newTestSignedRequestV4("GET", getGetObjectURL("", nilBucket, nilObject),
+		0, nil, "", "", nil)
+
+	if err != nil {
+		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
+	}
+	// execute the object layer set to `nil` test.
+	// `ExecObjectLayerAPINilTest` manages the operation.
+	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
+
+}
+
 // Wrapper for calling PutObject API handler tests using streaming signature v4 for both XL multiple disks and FS single drive setup.
 func TestAPIPutObjectStreamSigV4Handler(t *testing.T) {
 	defer DetectTestLeak(t)()
@@ -912,7 +1241,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Get Object end point.
 		req, err = newTestSignedRequestV4("PUT", getPutObjectURL("", testCase.bucketName, testCase.objectName),
-			int64(testCase.dataLen), bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey)
+			int64(testCase.dataLen), bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for Put Object: <ERROR> %v", i+1, err)
 		}
@@ -953,7 +1282,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for PUT Object endpoint.
 		reqV2, err = newTestSignedRequestV2("PUT", getPutObjectURL("", testCase.bucketName, testCase.objectName),
-			int64(testCase.dataLen), bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey)
+			int64(testCase.dataLen), bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: %s: Failed to create HTTP request for PutObject: <ERROR> %v", i+1, instanceType, err)
@@ -1013,7 +1342,7 @@ func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, a
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getPutObjectURL("", nilBucket, nilObject),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -1082,7 +1411,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 	}
 
 	a := 0
-	b := globalMinPartSize - 1
+	b := globalMinPartSize
 	var parts []CompletePart
 	for partNumber := 1; partNumber <= 2; partNumber++ {
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
@@ -1091,7 +1420,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 
 		// construct HTTP request for copy object.
 		var req *http.Request
-		req, err = newTestSignedRequestV4("PUT", cpPartURL, 0, nil, credentials.AccessKey, credentials.SecretKey)
+		req, err = newTestSignedRequestV4("PUT", cpPartURL, 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 		if err != nil {
 			t.Fatalf("Test failed to create HTTP request for copy object part: <ERROR> %v", err)
 		}
@@ -1102,7 +1431,7 @@ func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketNam
 
 		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
 		// Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request.
-		a = globalMinPartSize
+		a = globalMinPartSize + 1
 		b = len(bytesData[0].byteData) - 1
 		apiRouter.ServeHTTP(rec, req)
 		if rec.Code != http.StatusOK {
@@ -1373,11 +1702,11 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 		rec := httptest.NewRecorder()
 		if !testCase.invalidPartNumber || !testCase.maximumPartNumber {
 			// construct HTTP request for copy object.
-			req, err = newTestSignedRequestV4("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "1"), 0, nil, testCase.accessKey, testCase.secretKey)
+			req, err = newTestSignedRequestV4("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "1"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		} else if testCase.invalidPartNumber {
-			req, err = newTestSignedRequestV4("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "abc"), 0, nil, testCase.accessKey, testCase.secretKey)
+			req, err = newTestSignedRequestV4("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "abc"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		} else if testCase.maximumPartNumber {
-			req, err = newTestSignedRequestV4("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "99999"), 0, nil, testCase.accessKey, testCase.secretKey)
+			req, err = newTestSignedRequestV4("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "99999"), 0, nil, testCase.accessKey, testCase.secretKey, nil)
 		}
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
@@ -1447,7 +1776,7 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getCopyObjectPartURL("", nilBucket, nilObject, "0", "0"),
-		0, bytes.NewReader([]byte("testNilObjLayer")), "", "")
+		0, bytes.NewReader([]byte("testNilObjLayer")), "", "", nil)
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create http request for testing the response when object Layer is set to `nil`.", instanceType)
 	}
@@ -1740,7 +2069,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		rec := httptest.NewRecorder()
 		// construct HTTP request for copy object.
 		req, err = newTestSignedRequestV4("PUT", getCopyObjectURL("", testCase.bucketName, testCase.newObjectName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
@@ -1859,7 +2188,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getCopyObjectURL("", nilBucket, nilObject),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	// Below is how CopyObjectHandler is registered.
 	// bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?")
@@ -1890,7 +2219,7 @@ func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string
 	rec := httptest.NewRecorder()
 	// construct HTTP request for NewMultipart upload.
 	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, objectName),
-		0, nil, credentials.AccessKey, credentials.SecretKey)
+		0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 
 	if err != nil {
 		t.Fatalf("Failed to create HTTP request for NewMultipart Request: <ERROR> %v", err)
@@ -1923,7 +2252,7 @@ func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string
 	// construct HTTP request for NewMultipart upload.
 	// Setting an invalid accessID.
 	req, err = newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, objectName),
-		0, nil, "Invalid-AccessID", credentials.SecretKey)
+		0, nil, "Invalid-AccessID", credentials.SecretKey, nil)
 
 	if err != nil {
 		t.Fatalf("Failed to create HTTP request for NewMultipart Request: <ERROR> %v", err)
@@ -1942,7 +2271,7 @@ func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string
 	recV2 := httptest.NewRecorder()
 	// construct HTTP request for NewMultipartUpload endpoint.
 	reqV2, err := newTestSignedRequestV2("POST", getNewMultipartURL("", bucketName, objectName),
-		0, nil, credentials.AccessKey, credentials.SecretKey)
+		0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 
 	if err != nil {
 		t.Fatalf("Failed to create HTTP request for NewMultipart Request: <ERROR> %v", err)
@@ -1975,7 +2304,7 @@ func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string
 	// construct HTTP request for NewMultipartUpload endpoint.
 	// Setting invalid AccessID.
 	reqV2, err = newTestSignedRequestV2("POST", getNewMultipartURL("", bucketName, objectName),
-		0, nil, "Invalid-AccessID", credentials.SecretKey)
+		0, nil, "Invalid-AccessID", credentials.SecretKey, nil)
 
 	if err != nil {
 		t.Fatalf("Failed to create HTTP request for NewMultipart Request: <ERROR> %v", err)
@@ -2010,7 +2339,7 @@ func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("POST", getNewMultipartURL("", nilBucket, nilObject),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -2046,7 +2375,7 @@ func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketNam
 			defer wg.Done()
 			rec := httptest.NewRecorder()
 			// construct HTTP request NewMultipartUpload.
-			req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, objectName), 0, nil, credentials.AccessKey, credentials.SecretKey)
+			req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, objectName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 
 			if err != nil {
 				t.Fatalf("Failed to create HTTP request for NewMultipart request: <ERROR> %v", err)
@@ -2362,7 +2691,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 		}
 		// Indicating that all parts are uploaded and initiating CompleteMultipartUpload.
 		req, err = newTestSignedRequestV4("POST", getCompleteMultipartUploadURL("", bucketName, objectName, testCase.uploadID),
-			int64(len(completeBytes)), bytes.NewReader(completeBytes), testCase.accessKey, testCase.secretKey)
+			int64(len(completeBytes)), bytes.NewReader(completeBytes), testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for CompleteMultipartUpload: <ERROR> %v", err)
 		}
@@ -2423,7 +2752,7 @@ func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName s
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("POST", getCompleteMultipartUploadURL("", nilBucket, nilObject, "dummy-uploadID"),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -2547,7 +2876,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 		var req *http.Request
 		// Indicating that all parts are uploaded and initiating abortMultipartUpload.
 		req, err = newTestSignedRequestV4("DELETE", getAbortMultipartUploadURL("", testCase.bucket, testCase.object, testCase.uploadID),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for AbortMultipartUpload: <ERROR> %v", err)
 		}
@@ -2586,7 +2915,7 @@ func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName stri
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("DELETE", getAbortMultipartUploadURL("", nilBucket, nilObject, "dummy-uploadID"),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -2692,7 +3021,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 		rec := httptest.NewRecorder()
 		// construct HTTP request for Delete Object end point.
 		req, err = newTestSignedRequestV4("DELETE", getDeleteObjectURL("", testCase.bucketName, testCase.objectName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create HTTP request for Delete Object: <ERROR> %v", i+1, err)
@@ -2710,7 +3039,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 		recV2 := httptest.NewRecorder()
 		// construct HTTP request for Delete Object endpoint.
 		reqV2, err = newTestSignedRequestV2("DELETE", getDeleteObjectURL("", testCase.bucketName, testCase.objectName),
-			0, nil, testCase.accessKey, testCase.secretKey)
+			0, nil, testCase.accessKey, testCase.secretKey, nil)
 
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for NewMultipart Request: <ERROR> %v", err)
@@ -2747,7 +3076,7 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("DELETE", getDeleteObjectURL("", nilBucket, nilObject),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -2769,7 +3098,7 @@ func testAPIPutObjectPartHandlerPreSign(obj ObjectLayer, instanceType, bucketNam
 	testObject := "testobject"
 	rec := httptest.NewRecorder()
 	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
-		0, nil, credentials.AccessKey, credentials.SecretKey)
+		0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 	if err != nil {
 		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
 			instanceType, bucketName, testObject, err)
@@ -2836,7 +3165,7 @@ func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketN
 	testObject := "testobject"
 	rec := httptest.NewRecorder()
 	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
-		0, nil, credentials.AccessKey, credentials.SecretKey)
+		0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 	if err != nil {
 		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
 			instanceType, bucketName, testObject, err)
@@ -3107,7 +3436,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 			// constructing a v4 signed HTTP request.
 			reqV4, err = newTestSignedRequestV4("PUT",
 				getPutObjectPartURL("", bucketName, test.objectName, uploadID, test.partNumber),
-				0, test.reader, test.accessKey, test.secretKey)
+				0, test.reader, test.accessKey, test.secretKey, nil)
 			if err != nil {
 				t.Fatalf("Failed to create a signed V4 request to upload part for %s/%s: <ERROR> %v",
 					bucketName, test.objectName, err)
@@ -3116,7 +3445,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 			// construct HTTP request for PutObject Part Object endpoint.
 			reqV2, err = newTestSignedRequestV2("PUT",
 				getPutObjectPartURL("", bucketName, test.objectName, uploadID, test.partNumber),
-				0, test.reader, test.accessKey, test.secretKey)
+				0, test.reader, test.accessKey, test.secretKey, nil)
 
 			if err != nil {
 				t.Fatalf("Test %d %s Failed to create a V2  signed request to upload part for %s/%s: <ERROR> %v", i+1, instanceType,
@@ -3218,7 +3547,7 @@ func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName strin
 	nilObject := "dummy-object"
 
 	nilReq, err := newTestSignedRequestV4("PUT", getPutObjectPartURL("", nilBucket, nilObject, "0", "0"),
-		0, bytes.NewReader([]byte("testNilObjLayer")), "", "")
+		0, bytes.NewReader([]byte("testNilObjLayer")), "", "", nil)
 
 	if err != nil {
 		t.Errorf("Minio %s: Failed to create http request for testing the response when object Layer is set to `nil`.", instanceType)
@@ -3241,7 +3570,7 @@ func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketN
 	testObject := "testobject"
 	rec := httptest.NewRecorder()
 	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, testObject),
-		0, nil, credentials.AccessKey, credentials.SecretKey)
+		0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 	if err != nil {
 		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
 			instanceType, bucketName, testObject, err)
@@ -3264,7 +3593,7 @@ func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketN
 	rec = httptest.NewRecorder()
 	req, err = newTestSignedRequestV4("PUT",
 		getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
-		int64(len("hello")), bytes.NewReader([]byte("hello")), credentials.AccessKey, credentials.SecretKey)
+		int64(len("hello")), bytes.NewReader([]byte("hello")), credentials.AccessKey, credentials.SecretKey, nil)
 	if err != nil {
 		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
 			instanceType, bucketName, testObject, err)
@@ -3424,7 +3753,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
 			// constructing a v4 signed HTTP request for ListMultipartUploads.
 			reqV4, err = newTestSignedRequestV4("GET",
 				getListMultipartURLWithParams("", bucketName, testObject, uploadID, test.maxParts, test.partNumberMarker, ""),
-				0, nil, credentials.AccessKey, credentials.SecretKey)
+				0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 
 			if err != nil {
 				t.Fatalf("Failed to create a V4 signed request to list object parts for %s/%s: <ERROR> %v.",
@@ -3434,7 +3763,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
 			// construct HTTP request for PutObject Part Object endpoint.
 			reqV2, err = newTestSignedRequestV2("GET",
 				getListMultipartURLWithParams("", bucketName, testObject, uploadID, test.maxParts, test.partNumberMarker, ""),
-				0, nil, credentials.AccessKey, credentials.SecretKey)
+				0, nil, credentials.AccessKey, credentials.SecretKey, nil)
 
 			if err != nil {
 				t.Fatalf("Failed to create a V2 signed request to list object parts for %s/%s: <ERROR> %v.",
@@ -3522,7 +3851,7 @@ func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName str
 
 	nilReq, err := newTestSignedRequestV4("GET",
 		getListMultipartURLWithParams("", nilBucket, nilObject, "dummy-uploadID", "0", "0", ""),
-		0, nil, "", "")
+		0, nil, "", "", nil)
 	if err != nil {
 		t.Errorf("Minio %s:Failed to create http request for testing the response when object Layer is set to `nil`.", instanceType)
 	}

cmd/peer-rpc-client.go

@@ -165,9 +165,9 @@ func makeRemoteRPCClients(endpoints EndpointList) map[xnet.Host]*PeerRPCClient {
 	peerRPCClientMap := make(map[xnet.Host]*PeerRPCClient)
 	for _, hostStr := range GetRemotePeers(endpoints) {
 		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse peer RPC Host", context.Background())
+		logger.FatalIf(err, "Unable to parse peer RPC Host")
 		rpcClient, err := NewPeerRPCClient(host)
-		logger.FatalIf(err, "Unable to parse peer RPC Client", context.Background())
+		logger.FatalIf(err, "Unable to parse peer RPC Client")
 		peerRPCClientMap[*host] = rpcClient
 	}
 

cmd/peer-rpc-server.go

@@ -45,6 +45,11 @@ type DeleteBucketArgs struct {
 
 // DeleteBucket - handles delete bucket RPC call which removes all values of given bucket in global NotificationSys object.
 func (receiver *peerRPCReceiver) DeleteBucket(args *DeleteBucketArgs, reply *VoidReply) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	globalNotificationSys.RemoveNotification(args.BucketName)
 	globalPolicySys.Remove(args.BucketName)
 	return nil
@@ -59,6 +64,11 @@ type SetBucketPolicyArgs struct {
 
 // SetBucketPolicy - handles set bucket policy RPC call which adds bucket policy to globalPolicySys.
 func (receiver *peerRPCReceiver) SetBucketPolicy(args *SetBucketPolicyArgs, reply *VoidReply) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	globalPolicySys.Set(args.BucketName, args.Policy)
 	return nil
 }
@@ -71,6 +81,11 @@ type RemoveBucketPolicyArgs struct {
 
 // RemoveBucketPolicy - handles delete bucket policy RPC call which removes bucket policy to globalPolicySys.
 func (receiver *peerRPCReceiver) RemoveBucketPolicy(args *RemoveBucketPolicyArgs, reply *VoidReply) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	globalPolicySys.Remove(args.BucketName)
 	return nil
 }
@@ -84,6 +99,11 @@ type PutBucketNotificationArgs struct {
 
 // PutBucketNotification - handles put bucket notification RPC call which adds rules to given bucket to global NotificationSys object.
 func (receiver *peerRPCReceiver) PutBucketNotification(args *PutBucketNotificationArgs, reply *VoidReply) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	globalNotificationSys.AddRulesMap(args.BucketName, args.RulesMap)
 	return nil
 }
@@ -100,6 +120,11 @@ type ListenBucketNotificationArgs struct {
 
 // ListenBucketNotification - handles listen bucket notification RPC call. It creates PeerRPCClient target which pushes requested events to target in remote peer.
 func (receiver *peerRPCReceiver) ListenBucketNotification(args *ListenBucketNotificationArgs, reply *VoidReply) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	rpcClient := globalNotificationSys.GetPeerRPCClient(args.Addr)
 	if rpcClient == nil {
 		return fmt.Errorf("unable to find PeerRPCClient for provided address %v. This happens only if remote and this minio run with different set of endpoints", args.Addr)
@@ -126,6 +151,11 @@ type RemoteTargetExistArgs struct {
 
 // RemoteTargetExist - handles target ID exist RPC call which checks whether given target ID is a HTTP client target or not.
 func (receiver *peerRPCReceiver) RemoteTargetExist(args *RemoteTargetExistArgs, reply *bool) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	*reply = globalNotificationSys.RemoteTargetExist(args.BucketName, args.TargetID)
 	return nil
 }
@@ -140,6 +170,11 @@ type SendEventArgs struct {
 
 // SendEvent - handles send event RPC call which sends given event to target by given target ID.
 func (receiver *peerRPCReceiver) SendEvent(args *SendEventArgs, reply *bool) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	// Set default to true to keep the target.
 	*reply = true
 	errs := globalNotificationSys.send(args.BucketName, args.Event, args.TargetID)
@@ -159,6 +194,11 @@ func (receiver *peerRPCReceiver) SendEvent(args *SendEventArgs, reply *bool) err
 
 // LoadCredentials - handles load credentials RPC call.
 func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidReply) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(bucketConfigPrefix, minioConfigFile)
 	transactionConfigFile := configFile + ".transaction"
@@ -186,7 +226,7 @@ func NewPeerRPCServer() (*xrpc.Server, error) {
 // registerPeerRPCRouter - creates and registers Peer RPC server and its router.
 func registerPeerRPCRouter(router *mux.Router) {
 	rpcServer, err := NewPeerRPCServer()
-	logger.FatalIf(err, "Unable to initialize peer RPC Server", context.Background())
+	logger.FatalIf(err, "Unable to initialize peer RPC Server")
 	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
 	subrouter.Path(peerServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
 }

cmd/posix.go

@@ -918,7 +918,7 @@ func (s *posix) createFile(volume, path string) (f *os.File, err error) {
 // Currently we use fallocate when available to avoid disk fragmentation as much as possible
 func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 	// It doesn't make sense to create a negative-sized file
-	if fileSize <= 0 {
+	if fileSize < -1 {
 		return errInvalidArgument
 	}
 
@@ -949,8 +949,11 @@ func (s *posix) PrepareFile(volume, path string, fileSize int64) (err error) {
 	// Close upon return.
 	defer w.Close()
 
-	// Allocate needed disk space to append data
-	e := Fallocate(int(w.Fd()), 0, fileSize)
+	var e error
+	if fileSize > 0 {
+		// Allocate needed disk space to append data
+		e = Fallocate(int(w.Fd()), 0, fileSize)
+	}
 
 	// Ignore errors when Fallocate is not supported in the current system
 	if e != nil && !isSysErrNoSys(e) && !isSysErrOpNotSupported(e) {

cmd/prepare-storage.go

@@ -29,7 +29,7 @@ var printEndpointError = func() func(Endpoint, error) {
 	printOnce := make(map[Endpoint]map[string]bool)
 
 	return func(endpoint Endpoint, err error) {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.Host)
+		reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String())
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
 		m, ok := printOnce[endpoint]
 		if !ok {

cmd/rest/client.go

@@ -0,0 +1,108 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package rest
+
+import (
+	"context"
+	"crypto/tls"
+	"errors"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	xhttp "github.com/minio/minio/cmd/http"
+)
+
+// DefaultRESTTimeout - default RPC timeout is one minute.
+const DefaultRESTTimeout = 1 * time.Minute
+
+// Client - http based RPC client.
+type Client struct {
+	httpClient   *http.Client
+	url          *url.URL
+	newAuthToken func() string
+}
+
+// Call - make a REST call.
+func (c *Client) Call(method string, values url.Values, body io.Reader) (reply io.ReadCloser, err error) {
+	req, err := http.NewRequest(http.MethodPost, c.url.String()+"/"+method+"?"+values.Encode(), body)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("Authorization", "Bearer "+c.newAuthToken())
+	req.Header.Set("X-Minio-Time", time.Now().UTC().Format(time.RFC3339))
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		// Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
+		r := io.LimitReader(resp.Body, 1024)
+		b, err := ioutil.ReadAll(r)
+		if err != nil {
+			return nil, err
+		}
+		return nil, errors.New(string(b))
+	}
+	return resp.Body, nil
+}
+
+func newCustomDialContext(timeout time.Duration) func(ctx context.Context, network, addr string) (net.Conn, error) {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		dialer := &net.Dialer{
+			Timeout:   timeout,
+			KeepAlive: timeout,
+			DualStack: true,
+		}
+
+		conn, err := dialer.DialContext(ctx, network, addr)
+		if err != nil {
+			return nil, err
+		}
+
+		return xhttp.NewTimeoutConn(conn, timeout, timeout), nil
+	}
+}
+
+// NewClient - returns new RPC client.
+func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAuthToken func() string) *Client {
+	return &Client{
+		httpClient: &http.Client{
+			// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
+			// except custom DialContext and TLSClientConfig.
+			Transport: &http.Transport{
+				Proxy:                 http.ProxyFromEnvironment,
+				DialContext:           newCustomDialContext(timeout),
+				MaxIdleConnsPerHost:   4096,
+				MaxIdleConns:          4096,
+				IdleConnTimeout:       90 * time.Second,
+				TLSHandshakeTimeout:   10 * time.Second,
+				ExpectContinueTimeout: 1 * time.Second,
+				TLSClientConfig:       tlsConfig,
+				DisableCompression:    true,
+			},
+		},
+		url:          url,
+		newAuthToken: newAuthToken,
+	}
+}

cmd/routers.go

@@ -36,7 +36,7 @@ func newCacheObjectsFn() CacheObjectLayer {
 // Composed function registering routers for only distributed XL setup.
 func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 	// Register storage rpc router only if its a distributed setup.
-	registerStorageRPCRouters(router, endpoints)
+	registerStorageRESTHandlers(router, endpoints)
 
 	// Register distributed namespace lock.
 	registerDistNSLockRouter(router)

cmd/server_test.go

@@ -2728,6 +2728,8 @@ func (s *TestSuiteCommon) TestObjectMultipart(c *check) {
 	c.Assert(response.StatusCode, http.StatusOK)
 	var parts []CompletePart
 	for _, part := range completeUploads.Parts {
+		// For compressed objects, we dont treat E-Tag as checksum.
+		part.ETag = strings.Replace(part.ETag, "-1", "", -1)
 		part.ETag = canonicalizeETag(part.ETag)
 		parts = append(parts, part)
 	}

cmd/storage-rest-client.go

@@ -0,0 +1,343 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"crypto/tls"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/url"
+	"path"
+	"strconv"
+
+	"encoding/gob"
+	"encoding/hex"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/cmd/rest"
+	xnet "github.com/minio/minio/pkg/net"
+)
+
+func isNetworkDisconnectError(err error) bool {
+	if err == nil {
+		return false
+	}
+
+	if uerr, isURLError := err.(*url.Error); isURLError {
+		if uerr.Timeout() {
+			return true
+		}
+
+		err = uerr.Err
+	}
+
+	_, isNetOpError := err.(*net.OpError)
+	return isNetOpError
+}
+
+// Converts rpc.ServerError to underlying error. This function is
+// written so that the storageAPI errors are consistent across network
+// disks as well.
+func toStorageErr(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	if isNetworkDisconnectError(err) {
+		return errDiskNotFound
+	}
+
+	switch err.Error() {
+	case io.EOF.Error():
+		return io.EOF
+	case io.ErrUnexpectedEOF.Error():
+		return io.ErrUnexpectedEOF
+	case errUnexpected.Error():
+		return errUnexpected
+	case errDiskFull.Error():
+		return errDiskFull
+	case errVolumeNotFound.Error():
+		return errVolumeNotFound
+	case errVolumeExists.Error():
+		return errVolumeExists
+	case errFileNotFound.Error():
+		return errFileNotFound
+	case errFileNameTooLong.Error():
+		return errFileNameTooLong
+	case errFileAccessDenied.Error():
+		return errFileAccessDenied
+	case errIsNotRegular.Error():
+		return errIsNotRegular
+	case errVolumeNotEmpty.Error():
+		return errVolumeNotEmpty
+	case errVolumeAccessDenied.Error():
+		return errVolumeAccessDenied
+	case errCorruptedFormat.Error():
+		return errCorruptedFormat
+	case errUnformattedDisk.Error():
+		return errUnformattedDisk
+	case errInvalidAccessKeyID.Error():
+		return errInvalidAccessKeyID
+	case errAuthentication.Error():
+		return errAuthentication
+	case errRPCAPIVersionUnsupported.Error():
+		return errRPCAPIVersionUnsupported
+	case errServerTimeMismatch.Error():
+		return errServerTimeMismatch
+	}
+	return err
+}
+
+// Abstracts a remote disk.
+type storageRESTClient struct {
+	endpoint   Endpoint
+	restClient *rest.Client
+	connected  bool
+	lastError  error
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is makred disconnected
+// permanently. The only way to restore the storage connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *storageRESTClient) call(method string, values url.Values, body io.Reader) (respBody io.ReadCloser, err error) {
+	if !client.connected {
+		return nil, errDiskNotFound
+	}
+	respBody, err = client.restClient.Call(method, values, body)
+	if err == nil {
+		return respBody, nil
+	}
+	client.lastError = err
+	if isNetworkDisconnectError(err) {
+		client.connected = false
+	}
+
+	return nil, toStorageErr(err)
+}
+
+// Stringer provides a canonicalized representation of network device.
+func (client *storageRESTClient) String() string {
+	return client.endpoint.String()
+}
+
+// IsOnline - returns whether RPC client failed to connect or not.
+func (client *storageRESTClient) IsOnline() bool {
+	return client.connected
+}
+
+// LastError - returns the network error if any.
+func (client *storageRESTClient) LastError() error {
+	return client.lastError
+}
+
+// DiskInfo - fetch disk information for a remote disk.
+func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) {
+	respBody, err := client.call(storageRESTMethodDiskInfo, nil, nil)
+	if err != nil {
+		return
+	}
+	defer CloseResponse(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
+// MakeVol - create a volume on a remote disk.
+func (client *storageRESTClient) MakeVol(volume string) (err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	respBody, err := client.call(storageRESTMethodMakeVol, values, nil)
+	defer CloseResponse(respBody)
+	return err
+}
+
+// ListVols - List all volumes on a remote disk.
+func (client *storageRESTClient) ListVols() (volinfo []VolInfo, err error) {
+	respBody, err := client.call(storageRESTMethodListVols, nil, nil)
+	if err != nil {
+		return
+	}
+	defer CloseResponse(respBody)
+	err = gob.NewDecoder(respBody).Decode(&volinfo)
+	return volinfo, err
+}
+
+// StatVol - get volume info over the network.
+func (client *storageRESTClient) StatVol(volume string) (volInfo VolInfo, err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	respBody, err := client.call(storageRESTMethodStatVol, values, nil)
+	if err != nil {
+		return
+	}
+	defer CloseResponse(respBody)
+	err = gob.NewDecoder(respBody).Decode(&volInfo)
+	return volInfo, err
+}
+
+// DeleteVol - Deletes a volume over the network.
+func (client *storageRESTClient) DeleteVol(volume string) (err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	respBody, err := client.call(storageRESTMethodDeleteVol, values, nil)
+	defer CloseResponse(respBody)
+	return err
+}
+
+// PrepareFile - to fallocate() disk space for a file.
+func (client *storageRESTClient) PrepareFile(volume, path string, length int64) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	values.Set(storageRESTLength, strconv.Itoa(int(length)))
+	respBody, err := client.call(storageRESTMethodPrepareFile, values, nil)
+	defer CloseResponse(respBody)
+	return err
+}
+
+// AppendFile - append to a file.
+func (client *storageRESTClient) AppendFile(volume, path string, buffer []byte) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	reader := bytes.NewBuffer(buffer)
+	respBody, err := client.call(storageRESTMethodAppendFile, values, reader)
+	defer CloseResponse(respBody)
+	return err
+}
+
+// StatFile - stat a file.
+func (client *storageRESTClient) StatFile(volume, path string) (info FileInfo, err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	respBody, err := client.call(storageRESTMethodStatFile, values, nil)
+	if err != nil {
+		return info, err
+	}
+	defer CloseResponse(respBody)
+	err = gob.NewDecoder(respBody).Decode(&info)
+	return info, err
+}
+
+// ReadAll - reads all contents of a file.
+func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	respBody, err := client.call(storageRESTMethodReadAll, values, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer CloseResponse(respBody)
+	return ioutil.ReadAll(respBody)
+}
+
+// ReadFile - reads section of a file.
+func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	values.Set(storageRESTOffset, strconv.Itoa(int(offset)))
+	values.Set(storageRESTLength, strconv.Itoa(len(buffer)))
+	if verifier != nil {
+		values.Set(storageRESTBitrotAlgo, verifier.algorithm.String())
+		values.Set(storageRESTBitrotHash, hex.EncodeToString(verifier.sum))
+	} else {
+		values.Set(storageRESTBitrotAlgo, "")
+		values.Set(storageRESTBitrotHash, "")
+	}
+	respBody, err := client.call(storageRESTMethodReadFile, values, nil)
+	if err != nil {
+		return 0, err
+	}
+	defer CloseResponse(respBody)
+	n, err := io.ReadFull(respBody, buffer)
+	return int64(n), err
+}
+
+// ListDir - lists a directory.
+func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (entries []string, err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTDirPath, dirPath)
+	values.Set(storageRESTCount, strconv.Itoa(count))
+	respBody, err := client.call(storageRESTMethodListDir, values, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer CloseResponse(respBody)
+	err = gob.NewDecoder(respBody).Decode(&entries)
+	return entries, err
+}
+
+// DeleteFile - deletes a file.
+func (client *storageRESTClient) DeleteFile(volume, path string) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	respBody, err := client.call(storageRESTMethodDeleteFile, values, nil)
+	defer CloseResponse(respBody)
+	return err
+}
+
+// RenameFile - renames a file.
+func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
+	values := make(url.Values)
+	values.Set(storageRESTSrcVolume, srcVolume)
+	values.Set(storageRESTSrcPath, srcPath)
+	values.Set(storageRESTDstVolume, dstVolume)
+	values.Set(storageRESTDstPath, dstPath)
+	respBody, err := client.call(storageRESTMethodRenameFile, values, nil)
+	defer CloseResponse(respBody)
+	return err
+}
+
+// Close - marks the client as closed.
+func (client *storageRESTClient) Close() error {
+	client.connected = false
+	return nil
+}
+
+// Returns a storage rest client.
+func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
+	host, err := xnet.ParseHost(endpoint.Host)
+	logger.FatalIf(err, "Unable to parse storage Host")
+
+	scheme := "http"
+	if globalIsSSL {
+		scheme = "https"
+	}
+
+	serverURL := &url.URL{
+		Scheme: scheme,
+		Host:   endpoint.Host,
+		Path:   path.Join(storageRESTPath, endpoint.Path),
+	}
+
+	var tlsConfig *tls.Config
+	if globalIsSSL {
+		tlsConfig = &tls.Config{
+			ServerName: host.Name,
+			RootCAs:    globalRootCAs,
+		}
+	}
+
+	restClient := rest.NewClient(serverURL, tlsConfig, rest.DefaultRESTTimeout, newAuthToken)
+	return &storageRESTClient{endpoint: endpoint, restClient: restClient, connected: true}
+}

cmd/storage-rest-common.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+const storageRESTVersion = "v1"
+const storageRESTPath = minioReservedBucketPath + "/storage/" + storageRESTVersion + "/"
+
+const (
+	storageRESTMethodDiskInfo  = "diskinfo"
+	storageRESTMethodMakeVol   = "makevol"
+	storageRESTMethodStatVol   = "statvol"
+	storageRESTMethodDeleteVol = "deletevol"
+	storageRESTMethodListVols  = "listvols"
+
+	storageRESTMethodPrepareFile = "preparefile"
+	storageRESTMethodAppendFile  = "appendfile"
+	storageRESTMethodStatFile    = "statfile"
+	storageRESTMethodReadAll     = "readall"
+	storageRESTMethodReadFile    = "readfile"
+	storageRESTMethodListDir     = "listdir"
+	storageRESTMethodDeleteFile  = "deletefile"
+	storageRESTMethodRenameFile  = "renamefile"
+)
+
+const (
+	storageRESTVolume     = "volume"
+	storageRESTDirPath    = "dir-path"
+	storageRESTFilePath   = "file-path"
+	storageRESTSrcVolume  = "source-volume"
+	storageRESTSrcPath    = "source-path"
+	storageRESTDstVolume  = "destination-volume"
+	storageRESTDstPath    = "destination-path"
+	storageRESTOffset     = "offset"
+	storageRESTLength     = "length"
+	storageRESTCount      = "count"
+	storageRESTBitrotAlgo = "bitrot-algo"
+	storageRESTBitrotHash = "bitrot-hash"
+)

cmd/storage-rest-server.go

@@ -0,0 +1,353 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"fmt"
+	"io"
+	"path"
+	"strconv"
+
+	"net/http"
+
+	"encoding/gob"
+	"encoding/hex"
+
+	"time"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+)
+
+// To abstract a disk over network.
+type storageRESTServer struct {
+	storage *posix
+}
+
+func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
+// IsValid - To authenticate and verify the time difference.
+func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+	requestTimeStr := r.Header.Get("X-Minio-Time")
+	requestTime, err := time.Parse(time.RFC3339, requestTimeStr)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return false
+	}
+	utcNow := UTCNow()
+	delta := requestTime.Sub(utcNow)
+	if delta < 0 {
+		delta = delta * -1
+	}
+	if delta > DefaultSkewTime {
+		s.writeErrorResponse(w, fmt.Errorf("client time %v is too apart with server time %v", requestTime, utcNow))
+		return false
+	}
+	return true
+}
+
+// DiskInfoHandler - returns disk info.
+func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	info, err := s.storage.DiskInfo()
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	defer w.(http.Flusher).Flush()
+	gob.NewEncoder(w).Encode(info)
+}
+
+// MakeVolHandler - make a volume.
+func (s *storageRESTServer) MakeVolHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	err := s.storage.MakeVol(volume)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
+// ListVolsHandler - list volumes.
+func (s *storageRESTServer) ListVolsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	infos, err := s.storage.ListVols()
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	defer w.(http.Flusher).Flush()
+	gob.NewEncoder(w).Encode(&infos)
+}
+
+// StatVolHandler - stat a volume.
+func (s *storageRESTServer) StatVolHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	info, err := s.storage.StatVol(volume)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	defer w.(http.Flusher).Flush()
+	gob.NewEncoder(w).Encode(info)
+}
+
+// DeleteVolumeHandler - delete a volume.
+func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	err := s.storage.DeleteVol(volume)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
+// PrepareFileHandler - fallocate() space for a file.
+func (s *storageRESTServer) PrepareFileHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+	fileSizeStr := vars[storageRESTLength]
+	fileSize, err := strconv.Atoi(fileSizeStr)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	err = s.storage.PrepareFile(volume, filePath, int64(fileSize))
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
+// AppendFileHandler - append to a file.
+func (s *storageRESTServer) AppendFileHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+
+	if r.ContentLength < 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	buf := make([]byte, r.ContentLength)
+	_, err := io.ReadFull(r.Body, buf)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	err = s.storage.AppendFile(volume, filePath, buf)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
+// StatFileHandler - stat a file.
+func (s *storageRESTServer) StatFileHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+
+	info, err := s.storage.StatFile(volume, filePath)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	defer w.(http.Flusher).Flush()
+	gob.NewEncoder(w).Encode(info)
+}
+
+// ReadAllHandler - read all the contents of a file.
+func (s *storageRESTServer) ReadAllHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+
+	buf, err := s.storage.ReadAll(volume, filePath)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
+	w.Write(buf)
+}
+
+// ReadFileHandler - read section of a file.
+func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+	offset, err := strconv.Atoi(vars[storageRESTOffset])
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	length, err := strconv.Atoi(vars[storageRESTLength])
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	if offset < 0 || length < 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+	var verifier *BitrotVerifier
+	if vars[storageRESTBitrotAlgo] != "" {
+		hashStr := vars[storageRESTBitrotHash]
+		var hash []byte
+		hash, err = hex.DecodeString(hashStr)
+		if err != nil {
+			s.writeErrorResponse(w, err)
+			return
+		}
+		verifier = NewBitrotVerifier(BitrotAlgorithmFromString(vars[storageRESTBitrotAlgo]), hash)
+	}
+	buf := make([]byte, length)
+	_, err = s.storage.ReadFile(volume, filePath, int64(offset), buf, verifier)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	w.Header().Set("Content-Length", strconv.Itoa(len(buf)))
+	w.Write(buf)
+}
+
+// ListDirHandler - list a directory.
+func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	dirPath := vars[storageRESTDirPath]
+	count, err := strconv.Atoi(vars[storageRESTCount])
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	entries, err := s.storage.ListDir(volume, dirPath, count)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	defer w.(http.Flusher).Flush()
+	gob.NewEncoder(w).Encode(&entries)
+}
+
+// DeleteFileHandler - delete a file.
+func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+
+	err := s.storage.DeleteFile(volume, filePath)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
+// RenameFileHandler - rename a file.
+func (s *storageRESTServer) RenameFileHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	srcVolume := vars[storageRESTSrcVolume]
+	srcFilePath := vars[storageRESTSrcPath]
+	dstVolume := vars[storageRESTDstVolume]
+	dstFilePath := vars[storageRESTDstPath]
+	err := s.storage.RenameFile(srcVolume, srcFilePath, dstVolume, dstFilePath)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+	}
+}
+
+// registerStorageRPCRouter - register storage rpc router.
+func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
+	for _, endpoint := range endpoints {
+		if !endpoint.IsLocal {
+			continue
+		}
+		storage, err := newPosix(endpoint.Path)
+		if err != nil {
+			logger.Fatal(uiErrUnableToWriteInBackend(err), "Unable to initialize posix backend")
+		}
+
+		server := &storageRESTServer{storage}
+
+		subrouter := router.PathPrefix(path.Join(storageRESTPath, endpoint.Path)).Subrouter()
+
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler))
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler)).Queries(restQueries(storageRESTVolume)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodStatVol).HandlerFunc(httpTraceHdrs(server.StatVolHandler)).Queries(restQueries(storageRESTVolume)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodDeleteVol).HandlerFunc(httpTraceHdrs(server.DeleteVolHandler)).Queries(restQueries(storageRESTVolume)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodListVols).HandlerFunc(httpTraceHdrs(server.ListVolsHandler))
+
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodPrepareFile).HandlerFunc(httpTraceHdrs(server.PrepareFileHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTLength)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodAppendFile).HandlerFunc(httpTraceHdrs(server.AppendFileHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodStatFile).HandlerFunc(httpTraceHdrs(server.StatFileHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodReadAll).HandlerFunc(httpTraceHdrs(server.ReadAllHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodReadFile).HandlerFunc(httpTraceHdrs(server.ReadFileHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTOffset, storageRESTLength, storageRESTBitrotAlgo, storageRESTBitrotHash)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodListDir).HandlerFunc(httpTraceHdrs(server.ListDirHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTCount)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler)).
+			Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+		subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
+			Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
+
+	}
+}

cmd/storage-rest_test.go

@@ -18,18 +18,18 @@ package cmd
 
 import (
 	"io/ioutil"
-	"net/http"
 	"net/http/httptest"
 	"os"
 	"reflect"
 	"testing"
 
+	"github.com/gorilla/mux"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
 ///////////////////////////////////////////////////////////////////////////////
 //
-// Storage RPC server, storageRPCReceiver and StorageRPCClient are
+// Storage REST server, storageRESTReceiver and StorageRESTClient are
 // inter-dependent, below test functions are sufficient to test all of them.
 //
 ///////////////////////////////////////////////////////////////////////////////
@@ -520,182 +520,174 @@ func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) {
 	}
 }
 
-func newStorageRPCHTTPServerClient(t *testing.T) (*httptest.Server, *StorageRPCClient, *serverConfig, string) {
-	endpointPath, err := ioutil.TempDir("", ".TestStorageRPC.")
+func newStorageRESTHTTPServerClient(t *testing.T) (*httptest.Server, *storageRESTClient, *serverConfig, string) {
+	endpointPath, err := ioutil.TempDir("", ".TestStorageREST.")
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
 
-	rpcServer, err := NewStorageRPCServer(endpointPath)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rpcServer.ServeHTTP(w, r)
-	}))
+	router := mux.NewRouter()
+	httpServer := httptest.NewServer(router)
 
 	url, err := xnet.ParseURL(httpServer.URL)
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
+	url.Path = endpointPath
 
-	host, err := xnet.ParseHost(url.Host)
+	endpoint, err := NewEndpoint(url.String())
 	if err != nil {
-		t.Fatalf("unexpected error %v", err)
+		t.Fatalf("NewEndpoint failed %v", endpoint)
 	}
 
+	registerStorageRESTHandlers(router, EndpointList{endpoint})
+	restClient := newStorageRESTClient(endpoint)
+
 	prevGlobalServerConfig := globalServerConfig
 	globalServerConfig = newServerConfig()
 
-	rpcClient, err := NewStorageRPCClient(host, endpointPath)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-	rpcClient.connected = true
-
-	return httpServer, rpcClient, prevGlobalServerConfig, endpointPath
+	return httpServer, restClient, prevGlobalServerConfig, endpointPath
 }
 
-func TestStorageRPCClientDiskInfo(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientDiskInfo(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIDiskInfo(t, rpcClient)
+	testStorageAPIDiskInfo(t, restClient)
 }
 
-func TestStorageRPCClientMakeVol(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientMakeVol(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIMakeVol(t, rpcClient)
+	testStorageAPIMakeVol(t, restClient)
 }
 
-func TestStorageRPCClientListVols(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientListVols(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIListVols(t, rpcClient)
+	testStorageAPIListVols(t, restClient)
 }
 
-func TestStorageRPCClientStatVol(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientStatVol(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIStatVol(t, rpcClient)
+	testStorageAPIStatVol(t, restClient)
 }
 
-func TestStorageRPCClientDeleteVol(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientDeleteVol(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIDeleteVol(t, rpcClient)
+	testStorageAPIDeleteVol(t, restClient)
 }
 
-func TestStorageRPCClientStatFile(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientStatFile(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIStatFile(t, rpcClient)
+	testStorageAPIStatFile(t, restClient)
 }
 
-func TestStorageRPCClientListDir(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientListDir(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIListDir(t, rpcClient)
+	testStorageAPIListDir(t, restClient)
 }
 
-func TestStorageRPCClientReadAll(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientReadAll(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIReadAll(t, rpcClient)
+	testStorageAPIReadAll(t, restClient)
 }
 
-func TestStorageRPCClientReadFile(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientReadFile(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIReadFile(t, rpcClient)
+	testStorageAPIReadFile(t, restClient)
 }
 
-func TestStorageRPCClientPrepareFile(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientPrepareFile(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIPrepareFile(t, rpcClient)
+	testStorageAPIPrepareFile(t, restClient)
 }
 
-func TestStorageRPCClientAppendFile(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientAppendFile(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIAppendFile(t, rpcClient)
+	testStorageAPIAppendFile(t, restClient)
 }
 
-func TestStorageRPCClientDeleteFile(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientDeleteFile(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIDeleteFile(t, rpcClient)
+	testStorageAPIDeleteFile(t, restClient)
 }
 
-func TestStorageRPCClientRenameFile(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig, endpointPath := newStorageRPCHTTPServerClient(t)
+func TestStorageRESTClientRenameFile(t *testing.T) {
+	httpServer, restClient, prevGlobalServerConfig, endpointPath := newStorageRESTHTTPServerClient(t)
 	defer httpServer.Close()
 	defer func() {
 		globalServerConfig = prevGlobalServerConfig
 	}()
 	defer os.RemoveAll(endpointPath)
 
-	testStorageAPIRenameFile(t, rpcClient)
+	testStorageAPIRenameFile(t, restClient)
 }

cmd/storage-rpc-client.go

@@ -1,339 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"crypto/tls"
-	"io"
-	"net"
-	"net/url"
-	"path"
-	"strings"
-
-	"github.com/minio/minio/cmd/logger"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-func isNetworkDisconnectError(err error) bool {
-	if err == nil {
-		return false
-	}
-
-	if uerr, isURLError := err.(*url.Error); isURLError {
-		if uerr.Timeout() {
-			return true
-		}
-
-		err = uerr.Err
-	}
-
-	_, isNetOpError := err.(*net.OpError)
-	return isNetOpError
-}
-
-// Converts rpc.ServerError to underlying error. This function is
-// written so that the storageAPI errors are consistent across network
-// disks as well.
-func toStorageErr(err error) error {
-	if err == nil {
-		return nil
-	}
-
-	if isNetworkDisconnectError(err) {
-		return errDiskNotFound
-	}
-
-	switch err.Error() {
-	case io.EOF.Error():
-		return io.EOF
-	case io.ErrUnexpectedEOF.Error():
-		return io.ErrUnexpectedEOF
-	case errUnexpected.Error():
-		return errUnexpected
-	case errDiskFull.Error():
-		return errDiskFull
-	case errVolumeNotFound.Error():
-		return errVolumeNotFound
-	case errVolumeExists.Error():
-		return errVolumeExists
-	case errFileNotFound.Error():
-		return errFileNotFound
-	case errFileNameTooLong.Error():
-		return errFileNameTooLong
-	case errFileAccessDenied.Error():
-		return errFileAccessDenied
-	case errIsNotRegular.Error():
-		return errIsNotRegular
-	case errVolumeNotEmpty.Error():
-		return errVolumeNotEmpty
-	case errVolumeAccessDenied.Error():
-		return errVolumeAccessDenied
-	case errCorruptedFormat.Error():
-		return errCorruptedFormat
-	case errUnformattedDisk.Error():
-		return errUnformattedDisk
-	case errInvalidAccessKeyID.Error():
-		return errInvalidAccessKeyID
-	case errAuthentication.Error():
-		return errAuthentication
-	case errRPCAPIVersionUnsupported.Error():
-		return errRPCAPIVersionUnsupported
-	case errServerTimeMismatch.Error():
-		return errServerTimeMismatch
-	}
-	return err
-}
-
-// StorageRPCClient - storage RPC client.
-type StorageRPCClient struct {
-	*RPCClient
-	connected bool
-	// Plain error of the last RPC call
-	lastRPCError error
-}
-
-// Stringer provides a canonicalized representation of network device.
-func (client *StorageRPCClient) String() string {
-	url := client.ServiceURL()
-	// Remove the storage RPC path prefix, internal paths are meaningless. why?
-	url.Path = strings.TrimPrefix(url.Path, storageServicePath)
-	return url.String()
-}
-
-// LastError - returns the last RPC call result, nil or error if any
-func (client *StorageRPCClient) LastError() error {
-	return client.lastRPCError
-}
-
-// Close - closes underneath RPC client.
-func (client *StorageRPCClient) Close() error {
-	client.connected = false
-	return toStorageErr(client.RPCClient.Close())
-}
-
-// IsOnline - returns whether RPC client failed to connect or not.
-func (client *StorageRPCClient) IsOnline() bool {
-	return client.connected
-}
-
-func (client *StorageRPCClient) connect() {
-	err := client.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{})
-	client.lastRPCError = err
-	client.connected = err == nil
-}
-
-func (client *StorageRPCClient) call(handler string, args interface {
-	SetAuthArgs(args AuthArgs)
-}, reply interface{}) error {
-
-	if !client.connected {
-		return errDiskNotFound
-	}
-
-	err := client.Call(handler, args, reply)
-	client.lastRPCError = err
-	if err == nil {
-		return nil
-	}
-
-	if isNetworkDisconnectError(err) {
-		client.connected = false
-	}
-
-	return toStorageErr(err)
-}
-
-// DiskInfo - fetch disk information for a remote disk.
-func (client *StorageRPCClient) DiskInfo() (info DiskInfo, err error) {
-	err = client.call(storageServiceName+".DiskInfo", &AuthArgs{}, &info)
-	return info, err
-}
-
-// MakeVol - create a volume on a remote disk.
-func (client *StorageRPCClient) MakeVol(volume string) (err error) {
-	return client.call(storageServiceName+".MakeVol", &VolArgs{Vol: volume}, &VoidReply{})
-}
-
-// ListVols - List all volumes on a remote disk.
-func (client *StorageRPCClient) ListVols() ([]VolInfo, error) {
-	var reply []VolInfo
-	err := client.call(storageServiceName+".ListVols", &AuthArgs{}, &reply)
-	return reply, err
-}
-
-// StatVol - get volume info over the network.
-func (client *StorageRPCClient) StatVol(volume string) (volInfo VolInfo, err error) {
-	err = client.call(storageServiceName+".StatVol", &VolArgs{Vol: volume}, &volInfo)
-	return volInfo, err
-}
-
-// DeleteVol - Deletes a volume over the network.
-func (client *StorageRPCClient) DeleteVol(volume string) (err error) {
-	return client.call(storageServiceName+".DeleteVol", &VolArgs{Vol: volume}, &VoidReply{})
-}
-
-// File operations.
-
-// PrepareFile - calls PrepareFile RPC.
-func (client *StorageRPCClient) PrepareFile(volume, path string, length int64) (err error) {
-	args := PrepareFileArgs{
-		Vol:  volume,
-		Path: path,
-		Size: length,
-	}
-	reply := VoidReply{}
-
-	return client.call(storageServiceName+".PrepareFile", &args, &reply)
-}
-
-// AppendFile - append file writes buffer to a remote network path.
-func (client *StorageRPCClient) AppendFile(volume, path string, buffer []byte) (err error) {
-	args := AppendFileArgs{
-		Vol:    volume,
-		Path:   path,
-		Buffer: buffer,
-	}
-	reply := VoidReply{}
-
-	return client.call(storageServiceName+".AppendFile", &args, &reply)
-}
-
-// StatFile - get latest Stat information for a file at path.
-func (client *StorageRPCClient) StatFile(volume, path string) (fileInfo FileInfo, err error) {
-	err = client.call(storageServiceName+".StatFile", &StatFileArgs{Vol: volume, Path: path}, &fileInfo)
-	return fileInfo, err
-}
-
-// ReadAll - reads entire contents of the file at path until EOF, returns the
-// contents in a byte slice. Returns buf == nil if err != nil.
-// This API is meant to be used on files which have small memory footprint, do
-// not use this on large files as it would cause server to crash.
-func (client *StorageRPCClient) ReadAll(volume, path string) (buf []byte, err error) {
-	err = client.call(storageServiceName+".ReadAll", &ReadAllArgs{Vol: volume, Path: path}, &buf)
-	return buf, err
-}
-
-// ReadFile - reads a file at remote path and fills the buffer.
-func (client *StorageRPCClient) ReadFile(volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (m int64, err error) {
-	// Recover from any panic and return error.
-	defer func() {
-		if r := recover(); r != nil {
-			err = bytes.ErrTooLarge
-		}
-	}()
-
-	args := ReadFileArgs{
-		Vol:      volume,
-		Path:     path,
-		Offset:   offset,
-		Length:   int64(len(buffer)),
-		Verified: true, // mark read as verified by default
-	}
-	if verifier != nil {
-		args.Algo = verifier.algorithm
-		args.ExpectedHash = verifier.sum
-	}
-	var reply []byte
-
-	err = client.call(storageServiceName+".ReadFile", &args, &reply)
-
-	// Copy reply to buffer.
-	copy(buffer, reply)
-
-	// Return length of result, err if any.
-	return int64(len(reply)), err
-}
-
-// ListDir - list all entries at prefix.
-func (client *StorageRPCClient) ListDir(volume, path string, count int) (entries []string, err error) {
-	err = client.call(storageServiceName+".ListDir", &ListDirArgs{Vol: volume, Path: path, Count: count}, &entries)
-	return entries, err
-}
-
-// DeleteFile - Delete a file at path.
-func (client *StorageRPCClient) DeleteFile(volume, path string) (err error) {
-	args := DeleteFileArgs{
-		Vol:  volume,
-		Path: path,
-	}
-	reply := VoidReply{}
-
-	return client.call(storageServiceName+".DeleteFile", &args, &reply)
-}
-
-// RenameFile - rename a remote file from source to destination.
-func (client *StorageRPCClient) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
-	args := RenameFileArgs{
-		SrcVol:  srcVolume,
-		SrcPath: srcPath,
-		DstVol:  dstVolume,
-		DstPath: dstPath,
-	}
-	reply := VoidReply{}
-
-	return client.call(storageServiceName+".RenameFile", &args, &reply)
-}
-
-// NewStorageRPCClient - returns new storage RPC client.
-func NewStorageRPCClient(host *xnet.Host, endpointPath string) (*StorageRPCClient, error) {
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
-	serviceURL := &xnet.URL{
-		Scheme: scheme,
-		Host:   host.String(),
-		Path:   path.Join(storageServicePath, endpointPath),
-	}
-
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: host.Name,
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	rpcClient, err := NewRPCClient(
-		RPCClientArgs{
-			NewAuthTokenFunc: newAuthToken,
-			RPCVersion:       globalRPCAPIVersion,
-			ServiceName:      storageServiceName,
-			ServiceURL:       serviceURL,
-			TLSConfig:        tlsConfig,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return &StorageRPCClient{RPCClient: rpcClient}, nil
-}
-
-// Initialize new storage rpc client.
-func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
-	host, err := xnet.ParseHost(endpoint.Host)
-	logger.FatalIf(err, "Unable to parse storage RPC Host", context.Background())
-	rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
-	logger.FatalIf(err, "Unable to initialize storage RPC client", context.Background())
-	// Attempt first try connection and save error if any.
-	rpcClient.connect()
-	return rpcClient
-}

cmd/storage-rpc-server.go

@@ -1,233 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"io"
-	"path"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	xrpc "github.com/minio/minio/cmd/rpc"
-)
-
-const storageServiceName = "Storage"
-const storageServiceSubPath = "/storage"
-
-var storageServicePath = path.Join(minioReservedBucketPath, storageServiceSubPath)
-
-// storageRPCReceiver - Storage RPC receiver for storage RPC server
-type storageRPCReceiver struct {
-	local *posix
-}
-
-// VolArgs - generic volume args.
-type VolArgs struct {
-	AuthArgs
-	Vol string
-}
-
-/// Storage operations handlers.
-
-// Connect - authenticates remote connection.
-func (receiver *storageRPCReceiver) Connect(args *AuthArgs, reply *VoidReply) (err error) {
-	return args.Authenticate()
-}
-
-// DiskInfo - disk info handler is rpc wrapper for DiskInfo operation.
-func (receiver *storageRPCReceiver) DiskInfo(args *AuthArgs, reply *DiskInfo) (err error) {
-	*reply, err = receiver.local.DiskInfo()
-	return err
-}
-
-/// Volume operations handlers.
-
-// MakeVol - make vol handler is rpc wrapper for MakeVol operation.
-func (receiver *storageRPCReceiver) MakeVol(args *VolArgs, reply *VoidReply) error {
-	return receiver.local.MakeVol(args.Vol)
-}
-
-// ListVols - list vols handler is rpc wrapper for ListVols operation.
-func (receiver *storageRPCReceiver) ListVols(args *AuthArgs, reply *[]VolInfo) (err error) {
-	*reply, err = receiver.local.ListVols()
-	return err
-}
-
-// StatVol - stat vol handler is a rpc wrapper for StatVol operation.
-func (receiver *storageRPCReceiver) StatVol(args *VolArgs, reply *VolInfo) (err error) {
-	*reply, err = receiver.local.StatVol(args.Vol)
-	return err
-}
-
-// DeleteVol - delete vol handler is a rpc wrapper for
-// DeleteVol operation.
-func (receiver *storageRPCReceiver) DeleteVol(args *VolArgs, reply *VoidReply) error {
-	return receiver.local.DeleteVol(args.Vol)
-}
-
-/// File operations
-
-// StatFileArgs represents stat file RPC arguments.
-type StatFileArgs struct {
-	AuthArgs
-	Vol  string
-	Path string
-}
-
-// StatFile - stat file handler is rpc wrapper to stat file.
-func (receiver *storageRPCReceiver) StatFile(args *StatFileArgs, reply *FileInfo) (err error) {
-	*reply, err = receiver.local.StatFile(args.Vol, args.Path)
-	return err
-}
-
-// ListDirArgs represents list contents RPC arguments.
-type ListDirArgs struct {
-	AuthArgs
-	Vol   string
-	Path  string
-	Count int
-}
-
-// ListDir - list directory handler is rpc wrapper to list dir.
-func (receiver *storageRPCReceiver) ListDir(args *ListDirArgs, reply *[]string) (err error) {
-	*reply, err = receiver.local.ListDir(args.Vol, args.Path, args.Count)
-	return err
-}
-
-// ReadAllArgs represents read all RPC arguments.
-type ReadAllArgs struct {
-	AuthArgs
-	Vol  string
-	Path string
-}
-
-// ReadAll - read all handler is rpc wrapper to read all storage API.
-func (receiver *storageRPCReceiver) ReadAll(args *ReadAllArgs, reply *[]byte) (err error) {
-	*reply, err = receiver.local.ReadAll(args.Vol, args.Path)
-	return err
-}
-
-// ReadFileArgs represents read file RPC arguments.
-type ReadFileArgs struct {
-	AuthArgs
-	Vol          string
-	Path         string
-	Offset       int64
-	Length       int64
-	Algo         BitrotAlgorithm
-	ExpectedHash []byte
-	Verified     bool
-}
-
-// ReadFile - read file handler is rpc wrapper to read file.
-func (receiver *storageRPCReceiver) ReadFile(args *ReadFileArgs, reply *[]byte) error {
-	var verifier *BitrotVerifier
-	if !args.Verified {
-		verifier = NewBitrotVerifier(args.Algo, args.ExpectedHash)
-	}
-
-	buf := make([]byte, args.Length)
-	n, err := receiver.local.ReadFile(args.Vol, args.Path, args.Offset, buf, verifier)
-	// Ignore io.ErrEnexpectedEOF for short reads i.e. less content available than requested.
-	if err == io.ErrUnexpectedEOF {
-		err = nil
-	}
-
-	*reply = buf[0:n]
-	return err
-}
-
-// PrepareFileArgs represents append file RPC arguments.
-type PrepareFileArgs struct {
-	AuthArgs
-	Vol  string
-	Path string
-	Size int64
-}
-
-// PrepareFile - prepare file handler is rpc wrapper to prepare file.
-func (receiver *storageRPCReceiver) PrepareFile(args *PrepareFileArgs, reply *VoidReply) error {
-	return receiver.local.PrepareFile(args.Vol, args.Path, args.Size)
-}
-
-// AppendFileArgs represents append file RPC arguments.
-type AppendFileArgs struct {
-	AuthArgs
-	Vol    string
-	Path   string
-	Buffer []byte
-}
-
-// AppendFile - append file handler is rpc wrapper to append file.
-func (receiver *storageRPCReceiver) AppendFile(args *AppendFileArgs, reply *VoidReply) error {
-	return receiver.local.AppendFile(args.Vol, args.Path, args.Buffer)
-}
-
-// DeleteFileArgs represents delete file RPC arguments.
-type DeleteFileArgs struct {
-	AuthArgs
-	Vol  string
-	Path string
-}
-
-// DeleteFile - delete file handler is rpc wrapper to delete file.
-func (receiver *storageRPCReceiver) DeleteFile(args *DeleteFileArgs, reply *VoidReply) error {
-	return receiver.local.DeleteFile(args.Vol, args.Path)
-}
-
-// RenameFileArgs represents rename file RPC arguments.
-type RenameFileArgs struct {
-	AuthArgs
-	SrcVol  string
-	SrcPath string
-	DstVol  string
-	DstPath string
-}
-
-// RenameFile - rename file handler is rpc wrapper to rename file.
-func (receiver *storageRPCReceiver) RenameFile(args *RenameFileArgs, reply *VoidReply) error {
-	return receiver.local.RenameFile(args.SrcVol, args.SrcPath, args.DstVol, args.DstPath)
-}
-
-// NewStorageRPCServer - returns new storage RPC server.
-func NewStorageRPCServer(endpointPath string) (*xrpc.Server, error) {
-	storage, err := newPosix(endpointPath)
-	if err != nil {
-		return nil, err
-	}
-
-	rpcServer := xrpc.NewServer()
-	if err = rpcServer.RegisterName(storageServiceName, &storageRPCReceiver{storage}); err != nil {
-		return nil, err
-	}
-
-	return rpcServer, nil
-}
-
-// registerStorageRPCRouter - register storage rpc router.
-func registerStorageRPCRouters(router *mux.Router, endpoints EndpointList) {
-	for _, endpoint := range endpoints {
-		if endpoint.IsLocal {
-			rpcServer, err := NewStorageRPCServer(endpoint.Path)
-			if err != nil {
-				logger.Fatal(uiErrUnableToWriteInBackend(err), "Unable to configure one of server's RPC services")
-			}
-			subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-			subrouter.Path(path.Join(storageServiceSubPath, endpoint.Path)).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
-		}
-	}
-}

cmd/streaming-signature-v4.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/minio/minio/pkg/auth"
 	sha256 "github.com/minio/sha256-simd"
 )
 
@@ -41,10 +42,7 @@ const (
 )
 
 // getChunkSignature - get chunk signature.
-func getChunkSignature(seedSignature string, region string, date time.Time, hashedChunk string) string {
-	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-
+func getChunkSignature(cred auth.Credentials, seedSignature string, region string, date time.Time, hashedChunk string) string {
 	// Calculate string to sign.
 	stringToSign := signV4ChunkedAlgorithm + "\n" +
 		date.Format(iso8601Format) + "\n" +
@@ -66,10 +64,7 @@ func getChunkSignature(seedSignature string, region string, date time.Time, hash
 //     - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html
 // returns signature, error otherwise if the signature mismatches or any other
 // error while parsing and validating.
-func calculateSeedSignature(r *http.Request) (signature string, region string, date time.Time, errCode APIErrorCode) {
-	// Access credentials.
-	cred := globalServerConfig.GetCredential()
-
+func calculateSeedSignature(cred auth.Credentials, r *http.Request) (signature string, region string, date time.Time, errCode APIErrorCode) {
 	// Copy request.
 	req := *r
 
@@ -156,12 +151,17 @@ var errMalformedEncoding = errors.New("malformed chunked encoding")
 // NewChunkedReader is not needed by normal applications. The http package
 // automatically decodes chunking when reading response bodies.
 func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) {
-	seedSignature, region, seedDate, errCode := calculateSeedSignature(req)
+	// Access credentials.
+	cred := globalServerConfig.GetCredential()
+
+	seedSignature, region, seedDate, errCode := calculateSeedSignature(cred, req)
 	if errCode != ErrNone {
 		return nil, errCode
 	}
+
 	return &s3ChunkedReader{
 		reader:            bufio.NewReader(req.Body),
+		cred:              cred,
 		seedSignature:     seedSignature,
 		seedDate:          seedDate,
 		region:            region,
@@ -174,6 +174,7 @@ func newSignV4ChunkedReader(req *http.Request) (io.ReadCloser, APIErrorCode) {
 // AWS Signature V4 chunked reader.
 type s3ChunkedReader struct {
 	reader            *bufio.Reader
+	cred              auth.Credentials
 	seedSignature     string
 	seedDate          time.Time
 	region            string
@@ -298,7 +299,7 @@ func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) {
 			// Calculate the hashed chunk.
 			hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil))
 			// Calculate the chunk signature.
-			newSignature := getChunkSignature(cr.seedSignature, cr.region, cr.seedDate, hashedChunk)
+			newSignature := getChunkSignature(cr.cred, cr.seedSignature, cr.region, cr.seedDate, hashedChunk)
 			if !compareSignatureV4(cr.chunkSignature, newSignature) {
 				// Chunk signature doesn't match we return signature does not match.
 				cr.err = errSignatureMismatch

cmd/test-utils_test.go

@@ -32,6 +32,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"encoding/pem"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
@@ -136,7 +137,7 @@ func calculateSignedChunkLength(chunkDataSize int64) int64 {
 }
 
 func mustGetHashReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *hash.Reader {
-	hr, err := hash.NewReader(data, size, md5hex, sha256hex)
+	hr, err := hash.NewReader(data, size, md5hex, sha256hex, size)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1110,9 +1111,9 @@ const (
 
 func newTestSignedRequest(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, signer signerType) (*http.Request, error) {
 	if signer == signerV2 {
-		return newTestSignedRequestV2(method, urlStr, contentLength, body, accessKey, secretKey)
+		return newTestSignedRequestV2(method, urlStr, contentLength, body, accessKey, secretKey, nil)
 	}
-	return newTestSignedRequestV4(method, urlStr, contentLength, body, accessKey, secretKey)
+	return newTestSignedRequestV4(method, urlStr, contentLength, body, accessKey, secretKey, nil)
 }
 
 // Returns request with correct signature but with incorrect SHA256.
@@ -1139,7 +1140,7 @@ func newTestSignedBadSHARequest(method, urlStr string, contentLength int64, body
 }
 
 // Returns new HTTP request object signed with signature v2.
-func newTestSignedRequestV2(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) {
+func newTestSignedRequestV2(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, headers map[string]string) (*http.Request, error) {
 	req, err := newTestRequest(method, urlStr, contentLength, body)
 	if err != nil {
 		return nil, err
@@ -1151,6 +1152,10 @@ func newTestSignedRequestV2(method, urlStr string, contentLength int64, body io.
 		return req, nil
 	}
 
+	for k, v := range headers {
+		req.Header.Add(k, v)
+	}
+
 	err = signRequestV2(req, accessKey, secretKey)
 	if err != nil {
 		return nil, err
@@ -1160,7 +1165,7 @@ func newTestSignedRequestV2(method, urlStr string, contentLength int64, body io.
 }
 
 // Returns new HTTP request object signed with signature v4.
-func newTestSignedRequestV4(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) {
+func newTestSignedRequestV4(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, headers map[string]string) (*http.Request, error) {
 	req, err := newTestRequest(method, urlStr, contentLength, body)
 	if err != nil {
 		return nil, err
@@ -1171,6 +1176,10 @@ func newTestSignedRequestV4(method, urlStr string, contentLength int64, body io.
 		return req, nil
 	}
 
+	for k, v := range headers {
+		req.Header.Add(k, v)
+	}
+
 	err = signRequestV4(req, accessKey, secretKey)
 	if err != nil {
 		return nil, err
@@ -2332,3 +2341,101 @@ func TestToErrIsNil(t *testing.T) {
 		t.Errorf("Test expected error code to be ErrNone, failed instead provided %d", toAPIErrorCode(nil))
 	}
 }
+
+// Uploads an object using DummyDataGen directly via the http
+// handler. Each part in a multipart object is a new DummyDataGen
+// instance (so the part sizes are needed to reconstruct the whole
+// object). When `len(partSizes) == 1`, asMultipart is used to upload
+// the object as multipart with 1 part or as a regular single object.
+//
+// All upload failures are considered test errors - this function is
+// intended as a helper for other tests.
+func uploadTestObject(t *testing.T, apiRouter http.Handler, creds auth.Credentials, bucketName, objectName string,
+	partSizes []int64, metadata map[string]string, asMultipart bool) {
+
+	if len(partSizes) == 0 {
+		t.Fatalf("Cannot upload an object without part sizes")
+	}
+	if len(partSizes) > 1 {
+		asMultipart = true
+	}
+
+	checkRespErr := func(rec *httptest.ResponseRecorder, exp int) {
+		if rec.Code != exp {
+			b, err := ioutil.ReadAll(rec.Body)
+			t.Fatalf("Expected: %v, Got: %v, Body: %s, err: %v", exp, rec.Code, string(b), err)
+		}
+	}
+
+	if !asMultipart {
+		srcData := NewDummyDataGen(partSizes[0], 0)
+		req, err := newTestSignedRequestV4("PUT", getPutObjectURL("", bucketName, objectName),
+			partSizes[0], srcData, creds.AccessKey, creds.SecretKey, metadata)
+		if err != nil {
+			t.Fatalf("Unexpected err: %#v", err)
+		}
+		rec := httptest.NewRecorder()
+		apiRouter.ServeHTTP(rec, req)
+		checkRespErr(rec, http.StatusOK)
+	} else {
+		// Multipart upload - each part is a new DummyDataGen
+		// (so the part lengths are required to verify the
+		// object when reading).
+
+		// Initiate mp upload
+		reqI, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, objectName),
+			0, nil, creds.AccessKey, creds.SecretKey, metadata)
+		if err != nil {
+			t.Fatalf("Unexpected err: %#v", err)
+		}
+		rec := httptest.NewRecorder()
+		apiRouter.ServeHTTP(rec, reqI)
+		checkRespErr(rec, http.StatusOK)
+		decoder := xml.NewDecoder(rec.Body)
+		multipartResponse := &InitiateMultipartUploadResponse{}
+		err = decoder.Decode(multipartResponse)
+		if err != nil {
+			t.Fatalf("Error decoding the recorded response Body")
+		}
+		upID := multipartResponse.UploadID
+
+		// Upload each part
+		var cp []CompletePart
+		cumulativeSum := int64(0)
+		for i, partLen := range partSizes {
+			partID := i + 1
+			partSrc := NewDummyDataGen(partLen, cumulativeSum)
+			cumulativeSum += partLen
+			req, errP := newTestSignedRequestV4("PUT",
+				getPutObjectPartURL("", bucketName, objectName, upID, fmt.Sprintf("%d", partID)),
+				partLen, partSrc, creds.AccessKey, creds.SecretKey, metadata)
+			if errP != nil {
+				t.Fatalf("Unexpected err: %#v", errP)
+			}
+			rec = httptest.NewRecorder()
+			apiRouter.ServeHTTP(rec, req)
+			checkRespErr(rec, http.StatusOK)
+			etag := rec.Header().Get("ETag")
+			if etag == "" {
+				t.Fatalf("Unexpected empty etag")
+			}
+			cp = append(cp, CompletePart{partID, etag[1 : len(etag)-1]})
+		}
+
+		// Call CompleteMultipart API
+		compMpBody, err := xml.Marshal(CompleteMultipartUpload{Parts: cp})
+		if err != nil {
+			t.Fatalf("Unexpected err: %#v", err)
+		}
+		reqC, errP := newTestSignedRequestV4("POST",
+			getCompleteMultipartUploadURL("", bucketName, objectName, upID),
+			int64(len(compMpBody)), bytes.NewReader(compMpBody),
+			creds.AccessKey, creds.SecretKey, metadata)
+		if errP != nil {
+			t.Fatalf("Unexpected err: %#v", errP)
+		}
+		rec = httptest.NewRecorder()
+		apiRouter.ServeHTTP(rec, reqC)
+		checkRespErr(rec, http.StatusOK)
+	}
+}

cmd/typed-errors.go

@@ -73,3 +73,6 @@ var errFirstDiskWait = errors.New("Waiting on other disks")
 
 // error returned when a bucket already exists
 var errBucketAlreadyExists = errors.New("Your previous request to create the named bucket succeeded and you already own it")
+
+// error returned for a negative actual size.
+var errInvalidDecompressedSize = errors.New("Invalid Decompressed Size")

cmd/ui-errors.go

@@ -29,6 +29,12 @@ var (
 		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
 	)
 
+	uiErrInvalidDomainValue = newUIErrFn(
+		"Invalid domain value",
+		"Please check the passed value",
+		"Domain can only accept DNS compatible values.",
+	)
+
 	uiErrInvalidErasureSetSize = newUIErrFn(
 		"Invalid erasure set size",
 		"Please check the passed value",
@@ -191,4 +197,10 @@ Example 1:
 		"Please contact Minio at https://slack.minio.io",
 		"",
 	)
+
+	uiErrInvalidCompressionIncludesValue = newUIErrFn(
+		"Invalid compression include value",
+		"Please check the passed value",
+		"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_ATTR=\"A,B,C\"",
+	)
 )

cmd/utils.go

@@ -23,6 +23,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -30,6 +31,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"sync"
@@ -182,26 +184,70 @@ func contains(slice interface{}, elem interface{}) bool {
 	return false
 }
 
+// profilerWrapper is created becauses pkg/profiler doesn't
+// provide any API to calculate the profiler file path in the
+// disk since the name of this latter is randomly generated.
+type profilerWrapper struct {
+	stopFn func()
+	pathFn func() string
+}
+
+func (p profilerWrapper) Stop() {
+	p.stopFn()
+}
+
+func (p profilerWrapper) Path() string {
+	return p.pathFn()
+}
+
 // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
-func startProfiler(profiler string) interface {
+func startProfiler(profilerType, dirPath string) (interface {
 	Stop()
-} {
-	// Enable profiler if ``_MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
-	switch profiler {
-	case "cpu":
-		return profile.Start(profile.CPUProfile, profile.NoShutdownHook)
-	case "mem":
-		return profile.Start(profile.MemProfile, profile.NoShutdownHook)
-	case "block":
-		return profile.Start(profile.BlockProfile, profile.NoShutdownHook)
-	default:
-		return nil
+	Path() string
+}, error) {
+
+	var err error
+	if dirPath == "" {
+		dirPath, err = ioutil.TempDir("", "profile")
+		if err != nil {
+			return nil, err
+		}
 	}
+
+	var profiler interface {
+		Stop()
+	}
+
+	// Enable profiler, supported types are [cpu, mem, block].
+	switch profilerType {
+	case "cpu":
+		profiler = profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "mem":
+		profiler = profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "block":
+		profiler = profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "mutex":
+		profiler = profile.Start(profile.MutexProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	case "trace":
+		profiler = profile.Start(profile.TraceProfile, profile.NoShutdownHook, profile.ProfilePath(dirPath))
+	default:
+		return nil, errors.New("profiler type unknown")
+	}
+
+	return &profilerWrapper{
+		stopFn: profiler.Stop,
+		pathFn: func() string {
+			return filepath.Join(dirPath, profilerType+".pprof")
+		},
+	}, nil
 }
 
 // Global profiler to be used by service go-routine.
 var globalProfiler interface {
+	// Stop the profiler
 	Stop()
+	// Return the path of the profiling file
+	Path() string
 }
 
 // dump the request into a string in JSON format.
@@ -412,3 +458,13 @@ func CloseResponse(respBody io.ReadCloser) {
 		respBody.Close()
 	}
 }
+
+// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)
+// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"]
+func restQueries(keys ...string) []string {
+	var accumulator []string
+	for _, key := range keys {
+		accumulator = append(accumulator, key, "{"+key+":.*}")
+	}
+	return accumulator
+}

cmd/utils_test.go

@@ -228,8 +228,9 @@ func TestURL2BucketObjectName(t *testing.T) {
 
 // Add tests for starting and stopping different profilers.
 func TestStartProfiler(t *testing.T) {
-	if startProfiler("") != nil {
-		t.Fatal("Expected nil, but non-nil value returned for invalid profiler.")
+	_, err := startProfiler("", "")
+	if err == nil {
+		t.Fatal("Expected a non nil error, but nil error returned for invalid profiler.")
 	}
 }
 
@@ -469,3 +470,25 @@ func TestIsErrIgnored(t *testing.T) {
 		}
 	}
 }
+
+// Test queries()
+func TestQueries(t *testing.T) {
+	var testCases = []struct {
+		keys      []string
+		keyvalues []string
+	}{
+		{
+			[]string{"aaaa", "bbbb"},
+			[]string{"aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"},
+		},
+	}
+
+	for i, test := range testCases {
+		keyvalues := restQueries(test.keys...)
+		for j := range keyvalues {
+			if keyvalues[j] != test.keyvalues[j] {
+				t.Fatalf("test %d: keyvalues[%d] does not match", i+1, j)
+			}
+		}
+	}
+}

cmd/web-handlers.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/url"
 	"os"
@@ -29,9 +30,11 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
+	snappy "github.com/golang/snappy"
 	"github.com/gorilla/mux"
 	"github.com/gorilla/rpc/v2/json2"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
@@ -42,6 +45,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/dns"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/policy"
@@ -648,7 +652,37 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	hashReader, err := hash.NewReader(r.Body, size, "", "")
+	reader := r.Body
+	actualSize := size
+
+	if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
+		// Storing the compression metadata.
+		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
+		metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
+
+		pipeReader, pipeWriter := io.Pipe()
+		snappyWriter := snappy.NewWriter(pipeWriter)
+
+		var actualReader *hash.Reader
+		actualReader, err = hash.NewReader(reader, size, "", "", actualSize)
+		if err != nil {
+			writeWebErrorResponse(w, err)
+			return
+		}
+
+		go func() {
+			// Writing to the compressed writer.
+			_, cerr := io.CopyN(snappyWriter, actualReader, actualSize)
+			snappyWriter.Close()
+			pipeWriter.CloseWithError(cerr)
+		}()
+
+		// Set compression metrics.
+		size = -1 // Since compressed size is un-predictable.
+		reader = pipeReader
+	}
+
+	hashReader, err := hash.NewReader(reader, size, "", "", actualSize)
 	if err != nil {
 		writeWebErrorResponse(w, err)
 		return
@@ -668,17 +702,27 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
+	}
+
 	// Notify object created event.
 	sendEvent(eventArgs{
 		EventName:  event.ObjectCreatedPut,
 		BucketName: bucket,
 		Object:     objInfo,
 		ReqParams:  extractReqParams(r),
+		UserAgent:  r.UserAgent(),
+		Host:       host,
+		Port:       port,
 	})
 }
 
 // Download - file download handler.
 func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
+	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		writeWebErrorResponse(w, errServerNotInitialized)
@@ -703,31 +747,66 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+
 	opts := ObjectOptions{}
+	getObjectInfo := objectAPI.GetObjectInfo
 	getObject := objectAPI.GetObject
-	if web.CacheAPI() != nil {
-		getObject = web.CacheAPI().GetObject
-	}
-	getObjectInfo := objectAPI.GetObjectInfo
 	if web.CacheAPI() != nil {
 		getObjectInfo = web.CacheAPI().GetObjectInfo
+		getObject = web.CacheAPI().GetObject
 	}
 	objInfo, err := getObjectInfo(context.Background(), bucket, object, opts)
 	if err != nil {
 		writeWebErrorResponse(w, err)
 		return
 	}
-
-	if objectAPI.IsEncryptionSupported() {
-		if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
-			writeWebErrorResponse(w, err)
+	length := objInfo.Size
+	var actualSize int64
+	if objInfo.IsCompressed() {
+		// Read the decompressed size from the meta.json.
+		actualSize = objInfo.GetActualSize()
+		if actualSize < 0 {
 			return
 		}
 	}
+	if objectAPI.IsEncryptionSupported() {
+		if _, err = DecryptObjectInfo(objInfo, r.Header); err != nil {
+			writeWebErrorResponse(w, err)
+			return
+		}
+		if crypto.IsEncrypted(objInfo.UserDefined) {
+			length, _ = objInfo.DecryptedSize()
+		}
+	}
 	var startOffset int64
-	length := objInfo.Size
 	var writer io.Writer
-	writer = w
+	if objInfo.IsCompressed() {
+		// The decompress metrics are set.
+		snappyStartOffset := 0
+		snappyLength := actualSize
+
+		// Open a pipe for compression
+		// Where compressWriter is actually passed to the getObject
+		decompressReader, compressWriter := io.Pipe()
+		snappyReader := snappy.NewReader(decompressReader)
+
+		// The limit is set to the actual size.
+		responseWriter := ioutil.LimitedWriter(w, int64(snappyStartOffset), snappyLength)
+		wg.Add(1) //For closures.
+		go func() {
+			defer wg.Done()
+
+			// Finally, writes to the client.
+			_, perr := io.Copy(responseWriter, snappyReader)
+
+			// Close the compressWriter if the data is read already.
+			// Closing the pipe, releases the writer passed to the getObject.
+			compressWriter.CloseWithError(perr)
+		}()
+		writer = compressWriter
+	} else {
+		writer = w
+	}
 	if objectAPI.IsEncryptionSupported() && crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		// Response writer should be limited early on for decryption upto required length,
 		// additionally also skipping mod(offset)64KiB boundaries.
@@ -740,12 +819,17 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		}
 		w.Header().Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 	}
+
 	httpWriter := ioutil.WriteOnClose(writer)
 
 	// Add content disposition.
 	w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object)))
 
 	if err = getObject(context.Background(), bucket, object, 0, -1, httpWriter, "", opts); err != nil {
+		httpWriter.Close()
+		if objInfo.IsCompressed() {
+			wg.Wait()
+		}
 		/// No need to print error, response writer already written to.
 		return
 	}
@@ -755,6 +839,28 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	if objInfo.IsCompressed() {
+		// Wait for decompression go-routine to retire.
+		wg.Wait()
+	}
+
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
+	}
+
+	// Notify object accessed via a GET request.
+	sendEvent(eventArgs{
+		EventName:    event.ObjectAccessedGet,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         host,
+		Port:         port,
+	})
 }
 
 // DownloadZipArgs - Argument for downloading a bunch of files as a zip file.
@@ -768,6 +874,13 @@ type DownloadZipArgs struct {
 
 // Takes a list of objects and creates a zip file that sent as the response body.
 func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
+	// Get host and port from Request.RemoteAddr.
+	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
+	if err != nil {
+		host, port = "", ""
+	}
+
+	var wg sync.WaitGroup
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		writeWebErrorResponse(w, errServerNotInitialized)
@@ -815,6 +928,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 		getObjectInfo = web.CacheAPI().GetObjectInfo
 	}
 	opts := ObjectOptions{}
+	var length int64
 	for _, object := range args.Objects {
 		// Writes compressed object file to the response.
 		zipit := func(objectName string) error {
@@ -822,27 +936,64 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 			if err != nil {
 				return err
 			}
+			length = info.Size
 			if objectAPI.IsEncryptionSupported() {
-				if _, err = DecryptObjectInfo(&info, r.Header); err != nil {
+				if _, err = DecryptObjectInfo(info, r.Header); err != nil {
 					writeWebErrorResponse(w, err)
 					return err
 				}
+				if crypto.IsEncrypted(info.UserDefined) {
+					length, _ = info.DecryptedSize()
+				}
+			}
+			length = info.Size
+			var actualSize int64
+			if info.IsCompressed() {
+				// Read the decompressed size from the meta.json.
+				actualSize = info.GetActualSize()
+				// Set the info.Size to the actualSize.
+				info.Size = actualSize
 			}
 			header := &zip.FileHeader{
 				Name:               strings.TrimPrefix(objectName, args.Prefix),
 				Method:             zip.Deflate,
-				UncompressedSize64: uint64(info.Size),
-				UncompressedSize:   uint32(info.Size),
+				UncompressedSize64: uint64(length),
+				UncompressedSize:   uint32(length),
 			}
-			wr, err := archive.CreateHeader(header)
+			zipWriter, err := archive.CreateHeader(header)
 			if err != nil {
 				writeWebErrorResponse(w, errUnexpected)
 				return err
 			}
 			var startOffset int64
-			length := info.Size
 			var writer io.Writer
-			writer = wr
+
+			if info.IsCompressed() {
+				// The decompress metrics are set.
+				snappyStartOffset := 0
+				snappyLength := actualSize
+
+				// Open a pipe for compression
+				// Where compressWriter is actually passed to the getObject
+				decompressReader, compressWriter := io.Pipe()
+				snappyReader := snappy.NewReader(decompressReader)
+
+				// The limit is set to the actual size.
+				responseWriter := ioutil.LimitedWriter(zipWriter, int64(snappyStartOffset), snappyLength)
+				wg.Add(1) //For closures.
+				go func() {
+					defer wg.Done()
+					// Finally, writes to the client.
+					_, perr := io.Copy(responseWriter, snappyReader)
+
+					// Close the compressWriter if the data is read already.
+					// Closing the pipe, releases the writer passed to the getObject.
+					compressWriter.CloseWithError(perr)
+				}()
+				writer = compressWriter
+			} else {
+				writer = zipWriter
+			}
 			if objectAPI.IsEncryptionSupported() && crypto.S3.IsEncrypted(info.UserDefined) {
 				// Response writer should be limited early on for decryption upto required length,
 				// additionally also skipping mod(offset)64KiB boundaries.
@@ -855,6 +1006,11 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 			}
 			httpWriter := ioutil.WriteOnClose(writer)
 			if err = getObject(context.Background(), args.BucketName, objectName, 0, length, httpWriter, "", opts); err != nil {
+				httpWriter.Close()
+				if info.IsCompressed() {
+					// Wait for decompression go-routine to retire.
+					wg.Wait()
+				}
 				return err
 			}
 			if err = httpWriter.Close(); err != nil {
@@ -863,6 +1019,23 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 					return err
 				}
 			}
+			if info.IsCompressed() {
+				// Wait for decompression go-routine to retire.
+				wg.Wait()
+			}
+
+			// Notify object accessed via a GET request.
+			sendEvent(eventArgs{
+				EventName:    event.ObjectAccessedGet,
+				BucketName:   args.BucketName,
+				Object:       info,
+				ReqParams:    extractReqParams(r),
+				RespElements: extractRespElements(w),
+				UserAgent:    r.UserAgent(),
+				Host:         host,
+				Port:         port,
+			})
+
 			return nil
 		}
 

cmd/xl-sets.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
+	"net/http"
 	"sort"
 	"strings"
 	"sync"
@@ -193,6 +194,9 @@ func (s *xlSets) connectDisksWithQuorum() {
 			s.xlDisks[i][j] = disk
 			onlineDisks++
 		}
+		// Sleep for a while - so that we don't go into
+		// 100% CPU when half the disks are online.
+		time.Sleep(500 * time.Millisecond)
 	}
 }
 
@@ -518,6 +522,11 @@ func (s *xlSets) IsEncryptionSupported() bool {
 	return s.getHashedSet("").IsEncryptionSupported()
 }
 
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (s *xlSets) IsCompressionSupported() bool {
+	return s.getHashedSet("").IsCompressionSupported()
+}
+
 // DeleteBucket - deletes a bucket on all sets simultaneously,
 // even if one of the sets fail to delete buckets, we proceed to
 // undo a successful operation.
@@ -578,6 +587,11 @@ func (s *xlSets) ListBuckets(ctx context.Context) (buckets []BucketInfo, err err
 
 // --- Object Operations ---
 
+// GetObjectNInfo - returns object info and locked object ReadCloser
+func (s *xlSets) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+	return s.getHashedSet(object).GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+}
+
 // GetObject - reads an object from the hashedSet based on the object name.
 func (s *xlSets) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
 	return s.getHashedSet(object).GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
@@ -609,42 +623,14 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
 		return srcSet.CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 	}
 
-	// Hold write lock on destination since in both cases
-	// - if source and destination are same
-	// - if source and destination are different
-	// it is the sole mutating state.
-	objectDWLock := destSet.nsMutex.NewNSLock(destBucket, destObject)
-	if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
-		return objInfo, err
-	}
-	defer objectDWLock.Unlock()
-	// if source and destination are different, we have to hold
-	// additional read lock as well to protect against writes on
-	// source.
 	if !cpSrcDstSame {
-		// Hold read locks on source object only if we are
-		// going to read data from source object.
-		objectSRLock := srcSet.nsMutex.NewNSLock(srcBucket, srcObject)
-		if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
+		objectDWLock := destSet.nsMutex.NewNSLock(destBucket, destObject)
+		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
 			return objInfo, err
 		}
-		defer objectSRLock.RUnlock()
+		defer objectDWLock.Unlock()
 	}
 
-	go func() {
-		if gerr := srcSet.getObject(ctx, srcBucket, srcObject, 0, srcInfo.Size, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil {
-			if gerr = srcInfo.Writer.Close(); gerr != nil {
-				logger.LogIf(ctx, gerr)
-			}
-			return
-		}
-		// Close writer explicitly signaling we wrote all data.
-		if gerr := srcInfo.Writer.Close(); gerr != nil {
-			logger.LogIf(ctx, gerr)
-			return
-		}
-	}()
-
 	return destSet.putObject(ctx, destBucket, destObject, srcInfo.Reader, srcInfo.UserDefined, dstOpts)
 }
 
@@ -840,23 +826,8 @@ func (s *xlSets) NewMultipartUpload(ctx context.Context, bucket, object string,
 // Copies a part of an object from source hashedSet to destination hashedSet.
 func (s *xlSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int,
 	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (partInfo PartInfo, err error) {
-
-	srcSet := s.getHashedSet(srcObject)
 	destSet := s.getHashedSet(destObject)
 
-	go func() {
-		if gerr := srcSet.GetObject(ctx, srcBucket, srcObject, startOffset, length, srcInfo.Writer, srcInfo.ETag, srcOpts); gerr != nil {
-			if gerr = srcInfo.Writer.Close(); gerr != nil {
-				logger.LogIf(ctx, gerr)
-				return
-			}
-		}
-		if gerr := srcInfo.Writer.Close(); gerr != nil {
-			logger.LogIf(ctx, gerr)
-			return
-		}
-	}()
-
 	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, srcInfo.Reader, dstOpts)
 }
 

cmd/xl-v1-bucket.go

@@ -299,3 +299,8 @@ func (xl xlObjects) IsNotificationSupported() bool {
 func (xl xlObjects) IsEncryptionSupported() bool {
 	return true
 }
+
+// IsCompressionSupported returns whether compression is applicable for this layer.
+func (xl xlObjects) IsCompressionSupported() bool {
+	return true
+}

cmd/xl-v1-healing-common.go

@@ -158,17 +158,15 @@ func getLatestXLMeta(ctx context.Context, partsMetadata []xlMetaV1, errs []error
 //
 // - slice of errors about the state of data files on disk - can have
 //   a not-found error or a hash-mismatch error.
-//
-// - non-nil error if any of the disks failed unexpectedly (i.e. error
-//   other than file not found and not a checksum error).
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket,
-	object string) ([]StorageAPI, []error, error) {
+	object string) ([]StorageAPI, []error) {
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	buffer := []byte{}
 	dataErrs := make([]error, len(onlineDisks))
 
 	for i, onlineDisk := range onlineDisks {
 		if onlineDisk == nil {
+			dataErrs[i] = errDiskNotFound
 			continue
 		}
 
@@ -196,8 +194,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 				break
 			case hErr != nil:
 				logger.LogIf(ctx, hErr)
-				// abort on unhandled errors
-				return nil, nil, hErr
+				dataErrs[i] = hErr
+				break
 			}
 		}
 
@@ -207,5 +205,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 	}
 
-	return availableDisks, dataErrs, nil
+	return availableDisks, dataErrs
 }