Remove file added inadvertently (#7968)

Calling ListMultipartUploads fails if an upload is aborted while a
part is being uploaded because the directory for the upload exists
(since fsRenameFile ends up calling os.MkdirAll) but the meta JSON file
doesn't. To fix this we make sure an upload hasn't been aborted during
PutObjectPart by checking the existence of the directory for the upload
while moving the temporary part file into it.

.gitignore

@@ -9,7 +9,6 @@ site/
 /.idea/
 /Minio.iml
 **/access.log
-build/
 vendor/**/*.js
 vendor/**/*.json
 release
@@ -23,5 +22,4 @@ prime/
 stage/
 .sia_temp/
 config.json
-healthcheck
-check-user
+dockerscripts/healthcheck

.travis.yml

@@ -2,6 +2,14 @@ go_import_path: github.com/minio/minio
 
 language: go
 
+addons:
+  apt:
+    packages:
+    - shellcheck
+
+services:
+  - docker
+
 # this ensures PRs based on a local branch are not built twice
 # the downside is that a PR targeting a different branch is not built
 # but as a workaround you can add the branch to this list
@@ -26,7 +34,7 @@ matrix:
          - $HOME/gopath/pkg/mod
          - $HOME/go/pkg/mod
 
-      go: 1.12.1
+      go: 1.12.5
       script:
         - make
         - diff -au <(gofmt -s -d cmd) <(printf "")
@@ -37,12 +45,14 @@ matrix:
         - make verify
         - make coverage
         - cd browser && yarn && yarn test && cd ..
+        - bash -c 'shopt -s globstar; shellcheck mint/**/*.sh'
+
     - os: windows
       env:
         - ARCH=x86_64
         - CGO_ENABLED=0
         - GO111MODULE=on
-      go: 1.12.1
+      go: 1.12.5
       script:
         - go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
         - bash buildscripts/go-coverage.sh

CONTRIBUTING.md

@@ -37,7 +37,7 @@ After your code changes, make sure
 - To add test cases for the new code. If you have questions about how to do it, please ask on our [Slack](https://slack.min.io) channel.
 - To run `make verifiers`
 - To squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
-- To run `go test -race ./...` and `go build` completes.
+- To run `make test` and `make build` completes.
 
 ### Commit changes
 After verification, commit your changes. This is a [great post](https://chris.beams.io/posts/git-commit/) on how to write useful commit messages

Dockerfile

@@ -10,20 +10,19 @@ RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio && \
      go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
-     cd dockerscripts; go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go && \
-     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/check-user check-user.go
+     cd dockerscripts; go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
 
 FROM alpine:3.9
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 EXPOSE 9000
 
 COPY --from=0 /go/bin/minio /usr/bin/minio
 COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
-COPY --from=0 /usr/bin/check-user /usr/bin/check-user
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 RUN  \

Dockerfile.dev

@@ -2,20 +2,20 @@ FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck dockerscripts/check-user /usr/bin/
+COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck /usr/bin/
 COPY minio /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
      echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck && \
-     chmod +x /usr/bin/check-user
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 

Dockerfile.mint

@@ -0,0 +1,23 @@
+FROM ubuntu:16.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV LANG C.UTF-8
+
+ENV GOROOT /usr/local/go
+
+ENV GOPATH /usr/local
+
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+ENV MINT_ROOT_DIR /mint
+
+COPY mint /mint
+
+RUN apt-get --yes update && apt-get --yes upgrade && \
+    apt-get --yes --quiet install wget jq curl git dnsmasq && \
+    cd /mint && /mint/release.sh
+
+WORKDIR /mint
+
+ENTRYPOINT ["/mint/entrypoint.sh"]

Dockerfile.release

@@ -7,20 +7,19 @@ ENV GO111MODULE on
 RUN  \
      apk add --no-cache git && \
      git clone https://github.com/minio/minio && cd minio/dockerscripts && \
-     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go && \
-     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/check-user check-user.go
+     go build -tags kqueue -ldflags "-s -w" -o /usr/bin/healthcheck healthcheck.go
 
 FROM alpine:3.9
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=0 /usr/bin/healthcheck /usr/bin/healthcheck
-COPY --from=0 /usr/bin/check-user /usr/bin/check-user
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/
 
 ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
-    MINIO_SECRET_KEY_FILE=secret_key
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
 
 RUN \
      apk add --no-cache ca-certificates 'curl>7.61.0' 'su-exec>=0.2' && \
@@ -28,8 +27,7 @@ RUN \
      curl https://dl.min.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
      chmod +x /usr/bin/minio  && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/healthcheck && \
-     chmod +x /usr/bin/check-user
+     chmod +x /usr/bin/healthcheck
 
 EXPOSE 9000
 

Dockerfile.simpleci

@@ -33,7 +33,7 @@ USER ci
 RUN make
 RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
 RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
-RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
 RUN make verifiers
 RUN make crosscompile
 RUN make verify
@@ -56,15 +56,17 @@ FROM ubuntu:16.04
 
 COPY --from=0 /go/src/github.com/minio/minio/minio /usr/bin/minio
 COPY buildscripts/gateway-tests.sh /usr/bin/gateway-tests.sh
+COPY mint /mint
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV LANG C.UTF-8
 ENV GOROOT /usr/local/go
 ENV GOPATH /usr/local
 ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+ENV MINT_ROOT_DIR /mint
 
-RUN apt-get --yes update && apt-get --yes upgrade && apt-get --yes --quiet install wget jq curl git dnsmasq && \
-    git clone https://github.com/minio/mint.git /mint && \
+RUN apt-get --yes update && apt-get --yes upgrade && \
+    apt-get --yes --quiet install wget jq curl git dnsmasq && \
     cd /mint && /mint/release.sh
 
 WORKDIR /mint

Makefile

@@ -73,7 +73,6 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio 1>/dev/null
 	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/healthcheck $(PWD)/dockerscripts/healthcheck.go 1>/dev/null
-	@GO111MODULE=on GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/dockerscripts/check-user $(PWD)/dockerscripts/check-user.go 1>/dev/null
 
 docker: build
 	@docker build -t $(TAG) . -f Dockerfile.dev

browser/package-lock.json

@@ -5817,12 +5817,6 @@
           "integrity": "sha512-s5kLOcnH0XqDO+FvuaLX8DDjZ18CGFk7VygH40QoKPUQhW4e2rvM0rwUq0t8IQDOwYSeLK01U90OjzBTme2QqA==",
           "dev": true
         },
-        "lodash": {
-          "version": "4.17.11",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-          "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
-          "dev": true
-        },
         "micromatch": {
           "version": "3.1.10",
           "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-3.1.10.tgz",
@@ -6408,12 +6402,6 @@
           "requires": {
             "lodash": "^4.17.11"
           }
-        },
-        "lodash": {
-          "version": "4.17.11",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-          "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
-          "dev": true
         }
       }
     },
@@ -7645,14 +7633,14 @@
       }
     },
     "lodash": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-      "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg=="
+      "version": "4.17.14",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.14.tgz",
+      "integrity": "sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw=="
     },
     "lodash-es": {
-      "version": "4.17.11",
-      "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.11.tgz",
-      "integrity": "sha512-DHb1ub+rMjjrxqlB3H56/6MXtm1lSksDp2rA2cNWjG8mlDUYFhUj3Di2Zn5IwSU87xLv8tNIQ7sSwE/YOX/D/Q=="
+      "version": "4.17.14",
+      "resolved": "https://registry.npmjs.org/lodash-es/-/lodash-es-4.17.14.tgz",
+      "integrity": "sha512-7zchRrGa8UZXjD/4ivUWP1867jDkhzTG2c/uj739utSd7O/pFFdxspCemIFKEEjErbcqRzn8nKnGsi7mvTgRPA=="
     },
     "lodash._baseisequal": {
       "version": "3.0.7",
@@ -11078,14 +11066,6 @@
       "dev": true,
       "requires": {
         "lodash": "^4.17.11"
-      },
-      "dependencies": {
-        "lodash": {
-          "version": "4.17.11",
-          "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz",
-          "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==",
-          "dev": true
-        }
       }
     },
     "request-promise-native": {

browser/yarn.lock

@@ -5075,8 +5075,9 @@ locate-path@^3.0.0:
     path-exists "^3.0.0"
 
 lodash-es@^4.2.0, lodash-es@^4.2.1:
-  version "4.17.4"
-  resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.4.tgz#dcc1d7552e150a0640073ba9cb31d70f032950e7"
+  version "4.17.14"
+  resolved "https://registry.yarnpkg.com/lodash-es/-/lodash-es-4.17.14.tgz#12a95a963cc5955683cee3b74e85458954f37ecc"
+  integrity sha512-7zchRrGa8UZXjD/4ivUWP1867jDkhzTG2c/uj739utSd7O/pFFdxspCemIFKEEjErbcqRzn8nKnGsi7mvTgRPA==
 
 lodash._baseisequal@^3.0.0:
   version "3.0.7"
@@ -5185,14 +5186,10 @@ lodash.words@^3.0.0:
   dependencies:
     lodash._root "^3.0.0"
 
-lodash@^4.13.1, lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.4, lodash@^4.2.0, lodash@^4.2.1:
-  version "4.17.4"
-  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae"
-
-lodash@^4.17.3, lodash@^4.17.5, lodash@^4.3.0:
-  version "4.17.11"
-  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d"
-  integrity sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==
+lodash@^4.13.1, lodash@^4.14.0, lodash@^4.15.0, lodash@^4.17.3, lodash@^4.17.4, lodash@^4.17.5, lodash@^4.2.0, lodash@^4.2.1, lodash@^4.3.0:
+  version "4.17.14"
+  resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba"
+  integrity sha512-mmKYbW3GLuJeX+iGP+Y7Gp1AiGHGbXHCOh/jZmrawMmsE7MS4znI3RL2FsjbqOyMayHInjOeykW7PEajUk1/xw==
 
 loglevel@^1.4.1:
   version "1.6.1"

cmd/admin-handlers.go

@@ -241,17 +241,12 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	if objectAPI == nil {
 		return
 	}
-	hostName, err := getHostName(r)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 
 	serverInfo := globalNotificationSys.ServerInfo(ctx)
 	// Once we have received all the ServerInfo from peers
 	// add the local peer server info as well.
 	serverInfo = append(serverInfo, ServerInfo{
-		Addr: hostName,
+		Addr: getHostName(r),
 		Data: &ServerInfoData{
 			StorageInfo: objectAPI.StorageInfo(ctx),
 			ConnStats:   globalConnStats.toServerConnStats(),
@@ -442,18 +437,13 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
 	}
-	hostName, err := getHostName(r)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
 
 	peerLocks := globalNotificationSys.GetLocks(ctx)
 	// Once we have received all the locks currently used from peers
 	// add the local peer locks list as well.
 	localLocks := globalLockServer.ll.DupLockMap()
 	peerLocks = append(peerLocks, &PeerLocks{
-		Addr:  hostName,
+		Addr:  getHostName(r),
 		Locks: localLocks,
 	})
 
@@ -681,12 +671,12 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					// Start writing response to client
 					started = true
 					setCommonHeaders(w)
-					w.Header().Set(xhttp.ContentType, string(mimeJSON))
+					w.Header().Set(xhttp.ContentType, "text/event-stream")
 					// Set 200 OK status
 					w.WriteHeader(200)
 				}
 				// Send whitespace and keep connection open
-				w.Write([]byte("\n\r"))
+				w.Write([]byte(" "))
 				w.(http.Flusher).Flush()
 			case hr := <-respCh:
 				switch hr.apiErr {
@@ -1475,6 +1465,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HTTPTrace")
 	trcAll := r.URL.Query().Get("all") == "true"
+	trcErr := r.URL.Query().Get("err") == "true"
 
 	// Validate request signature.
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
@@ -1486,7 +1477,8 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	// Avoid reusing tcp connection if read timeout is hit
 	// This is needed to make r.Context().Done() work as
 	// expected in case of read timeout
-	w.Header().Add(xhttp.Connection, "close")
+	w.Header().Set(xhttp.Connection, "close")
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
 
 	doneCh := make(chan struct{})
 	defer close(doneCh)
@@ -1496,11 +1488,15 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	traceCh := make(chan interface{}, 4000)
 
 	filter := func(entry interface{}) bool {
+		trcInfo := entry.(trace.Info)
+		if trcErr && isHTTPStatusOK(trcInfo.RespInfo.StatusCode) {
+			return false
+		}
 		if trcAll {
 			return true
 		}
-		trcInfo := entry.(trace.Info)
 		return !strings.HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath)
+
 	}
 	remoteHosts := getRemoteHosts(globalEndpoints)
 	peers, err := getRestClients(remoteHosts)
@@ -1510,7 +1506,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	globalHTTPTrace.Subscribe(traceCh, doneCh, filter)
 
 	for _, peer := range peers {
-		peer.Trace(traceCh, doneCh, trcAll)
+		peer.Trace(traceCh, doneCh, trcAll, trcErr)
 	}
 
 	enc := json.NewEncoder(w)

cmd/admin-handlers_test.go

@@ -91,7 +91,9 @@ var (
         "enable": false,
         "format": "namespace",
         "url": "",
-        "index": ""
+        "index": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "kafka": {
@@ -150,6 +152,8 @@ var (
         "token": "",
         "secure": false,
         "pingInterval": 0,
+        "queueDir": "",
+        "queueLimit": 0,
         "streaming": {
           "enable": false,
           "clusterID": "",
@@ -166,7 +170,9 @@ var (
         "tls": {
 			"enable": false,
 			"skipVerify": false
-		}
+		},
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "postgresql": {
@@ -188,7 +194,9 @@ var (
         "format": "namespace",
         "address": "",
         "password": "",
-        "key": ""
+        "key": "",
+        "queueDir": "",
+        "queueLimit": 0
       }
     },
     "webhook": {

cmd/api-errors.go

@@ -92,6 +92,7 @@ const (
 	ErrMissingRequestBodyError
 	ErrNoSuchBucket
 	ErrNoSuchBucketPolicy
+	ErrNoSuchBucketLifecycle
 	ErrNoSuchKey
 	ErrNoSuchUpload
 	ErrNoSuchVersion
@@ -139,6 +140,7 @@ const (
 	ErrSlowDown
 	ErrInvalidPrefixMarker
 	ErrBadRequest
+	ErrKeyTooLongError
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -187,6 +189,7 @@ const (
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
 	ErrInvalidObjectName
+	ErrInvalidObjectNamePrefixSlash
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
@@ -465,6 +468,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket policy does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrNoSuchBucketLifecycle: {
+		Code:           "NoSuchBucketLifecycle",
+		Description:    "The bucket lifecycle configuration does not exist",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrNoSuchKey: {
 		Code:           "NoSuchKey",
 		Description:    "The specified key does not exist.",
@@ -682,6 +690,11 @@ var errorCodes = errorCodeMap{
 		Description:    "400 BadRequest",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrKeyTooLongError: {
+		Code:           "KeyTooLongError",
+		Description:    "Your key is too long",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 
 	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
 	ErrUnsignedHeaders: {
@@ -885,6 +898,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Object name contains unsupported characters.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidObjectNamePrefixSlash: {
+		Code:           "XMinioInvalidObjectName",
+		Description:    "Object name contains a leading slash.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidResourceName: {
 		Code:           "XMinioInvalidResourceName",
 		Description:    "Resource name contains bad components such as \"..\" or \".\".",
@@ -1579,6 +1597,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrMethodNotAllowed
 	case ObjectNameInvalid:
 		apiErr = ErrInvalidObjectName
+	case ObjectNamePrefixAsSlash:
+		apiErr = ErrInvalidObjectNamePrefixSlash
 	case InvalidUploadID:
 		apiErr = ErrNoSuchUpload
 	case InvalidPart:
@@ -1613,6 +1633,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrUnsupportedMetadata
 	case BucketPolicyNotFound:
 		apiErr = ErrNoSuchBucketPolicy
+	case BucketLifecycleNotFound:
+		apiErr = ErrNoSuchBucketLifecycle
 	case *event.ErrInvalidEventName:
 		apiErr = ErrEventNotification
 	case *event.ErrInvalidARN:
@@ -1639,6 +1661,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrBackendDown
 	case crypto.Error:
 		apiErr = ErrObjectTampered
+	case ObjectNameTooLong:
+		apiErr = ErrKeyTooLongError
 	default:
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648

cmd/api-response.go

@@ -582,7 +582,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
-		if browser {
+		if browser && globalIsBrowserEnabled {
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return

cmd/api-router.go

@@ -91,7 +91,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// GetBucketLocation
 		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
 		// GetBucketPolicy
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
+		// GetBucketLifecycle
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLifecycleHandler)).Queries("lifecycle", "")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
@@ -128,9 +130,12 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// ListObjectsV2
 		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
 		// ListObjectsV1 (Legacy)
-		bucket.Methods(http.MethodGet).HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
+		// PutBucketLifecycle
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketLifecycleHandler)).Queries("lifecycle", "")
 		// PutBucketPolicy
-		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
+
 		// PutBucketNotification
 		bucket.Methods(http.MethodPut).HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
 		// PutBucket
@@ -142,7 +147,9 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		// DeleteMultipleObjects
 		bucket.Methods(http.MethodPost).HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
 		// DeleteBucketPolicy
-		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
+		// DeleteBucketLifecycle
+		bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketLifecycleHandler)).Queries("lifecycle", "")
 		// DeleteBucket
 		bucket.Methods(http.MethodDelete).HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
 	}

cmd/bucket-handlers.go

@@ -537,7 +537,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Remove all tmp files creating during multipart upload
+	// Remove all tmp files created during multipart upload
 	defer form.RemoveAll()
 
 	// Extract all form fields
@@ -800,6 +800,8 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	globalNotificationSys.RemoveNotification(bucket)
 	globalPolicySys.Remove(bucket)
 	globalNotificationSys.DeleteBucket(ctx, bucket)
+	globalLifecycleSys.Remove(bucket)
+	globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
 
 	// Write success response.
 	writeSuccessNoContent(w)

cmd/bucket-lifecycle-handler.go

@@ -0,0 +1,164 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"encoding/xml"
+	"io"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
+)
+
+const (
+	// Lifecycle configuration file.
+	bucketLifecycleConfig = "lifecycle.xml"
+)
+
+// PutBucketLifecycleHandler - This HTTP handler stores given bucket lifecycle configuration as per
+// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html
+func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketLifecycle")
+
+	defer logger.AuditLog(w, r, "PutBucketLifecycle", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, lifecycle.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// PutBucketLifecycle always needs a Content-Md5
+	if _, ok := r.Header["Content-Md5"]; !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err = objAPI.SetBucketLifecycle(ctx, bucket, bucketLifecycle); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	globalLifecycleSys.Set(bucket, *bucketLifecycle)
+	globalNotificationSys.SetBucketLifecycle(ctx, bucket, bucketLifecycle)
+
+	// Success.
+	writeSuccessNoContent(w)
+}
+
+// GetBucketLifecycleHandler - This HTTP handler returns bucket policy configuration.
+func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketLifecycle")
+
+	defer logger.AuditLog(w, r, "GetBucketLifecycle", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, lifecycle.GetBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Read bucket access lifecycle.
+	bucketLifecycle, err := objAPI.GetBucketLifecycle(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	lifecycleData, err := xml.Marshal(bucketLifecycle)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Write lifecycle configuration to client.
+	writeSuccessResponseXML(w, lifecycleData)
+}
+
+// DeleteBucketLifecycleHandler - This HTTP handler removes bucket lifecycle configuration.
+func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketLifecycle")
+
+	defer logger.AuditLog(w, r, "DeleteBucketLifecycle", mustGetClaimsFromToken(r))
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	if s3Error := checkRequestAuthType(ctx, r, lifecycle.DeleteBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if err := objAPI.DeleteBucketLifecycle(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	globalLifecycleSys.Remove(bucket)
+	globalNotificationSys.RemoveBucketLifecycle(ctx, bucket)
+
+	// Success.
+	writeSuccessNoContent(w)
+}

cmd/bucket-notification-handlers.go

@@ -23,6 +23,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/event/target"
@@ -246,6 +247,8 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
 		return
 	}
 
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
+
 	target, err := target.NewHTTPClientTarget(*host, w)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/config-current.go

@@ -314,7 +314,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewElasticsearchTarget(k, v)
+		t, err := target.NewElasticsearchTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("elasticsearch(%s): %s", k, err.Error())
 		}
@@ -358,7 +358,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNATSTarget(k, v)
+		t, err := target.NewNATSTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("nats(%s): %s", k, err.Error())
 		}
@@ -369,7 +369,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewNSQTarget(k, v)
+		t, err := target.NewNSQTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("nsq(%s): %s", k, err.Error())
 		}
@@ -391,7 +391,7 @@ func (s *serverConfig) TestNotificationTargets() error {
 		if !v.Enable {
 			continue
 		}
-		t, err := target.NewRedisTarget(k, v)
+		t, err := target.NewRedisTarget(k, v, GlobalServiceDoneCh)
 		if err != nil {
 			return fmt.Errorf("redis(%s): %s", k, err.Error())
 		}
@@ -651,7 +651,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Elasticsearch {
 		if args.Enable {
-			newTarget, err := target.NewElasticsearchTarget(id, args)
+			newTarget, err := target.NewElasticsearchTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -710,7 +710,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NATS {
 		if args.Enable {
-			newTarget, err := target.NewNATSTarget(id, args)
+			newTarget, err := target.NewNATSTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -724,7 +724,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.NSQ {
 		if args.Enable {
-			newTarget, err := target.NewNSQTarget(id, args)
+			newTarget, err := target.NewNSQTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue
@@ -752,7 +752,7 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
 
 	for id, args := range config.Notify.Redis {
 		if args.Enable {
-			newTarget, err := target.NewRedisTarget(id, args)
+			newTarget, err := target.NewRedisTarget(id, args, GlobalServiceDoneCh)
 			if err != nil {
 				logger.LogIf(context.Background(), err)
 				continue

cmd/config-current_test.go

@@ -188,7 +188,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
 
 		// Test 12 - Test NATS
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "queueDir": "", "queueLimit": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
 
 		// Test 13 - Test ElasticSearch
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
@@ -224,10 +224,10 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
 
 		// Test 24 - Test valid Format for ElasticSearch
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex", "queueDir": "", "queueLimit": 0 } }}}`, true},
 
 		// Test 25 - Test Format for Redis
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1", "queueDir": "", "queueLimit": 0 } }}}`, false},
 
 		// Test 26 - Test valid Format for Redis
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
@@ -236,7 +236,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "username": "", "password": "", "queueDir": "", "queueLimit": 0}}}}`, false},
 
 		// Test 28 - Test NSQ
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": "", "queueDir": "", "queueLimit": 0} }}}`, false},
 	}
 
 	for i, testCase := range testCases {

cmd/config-versions.go

@@ -22,7 +22,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event/target"
-	"github.com/minio/minio/pkg/iam/policy"
+	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/iam/validator"
 	"github.com/minio/minio/pkg/quick"
 )
@@ -884,7 +884,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in notification targets.
+// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/daily-sweeper.go

@@ -119,9 +119,9 @@ func dailySweeper() {
 		break
 	}
 
-	// Perform a sweep round each 24 hours
+	// Perform a sweep round each month
 	for {
-		if time.Since(lastSweepTime) < 24*time.Hour {
+		if time.Since(lastSweepTime) < 30*24*time.Hour {
 			time.Sleep(time.Hour)
 			continue
 		}

cmd/disk-cache.go

@@ -945,7 +945,10 @@ func checkAtimeSupport(dir string) (err error) {
 	if err != nil {
 		return
 	}
-	if _, err = io.Copy(ioutil.Discard, file); err != io.EOF {
+	// add a sleep to ensure atime change is detected
+	time.Sleep(10 * time.Millisecond)
+
+	if _, err = io.Copy(ioutil.Discard, file); err != nil {
 		return
 	}
 

cmd/disk-cache_test.go

@@ -124,17 +124,10 @@ func TestGetCacheFSMaxUse(t *testing.T) {
 
 // test wildcard patterns for excluding entries from cache
 func TestCacheExclusion(t *testing.T) {
-	fsDirs, err := getRandomDisks(1)
-	if err != nil {
-		t.Fatal(err)
+	cobjects := &cacheObjects{
+		cache: nil,
 	}
-	cconfig := CacheConfig{Expiry: 30, Drives: fsDirs}
-	cobjects, err := newServerCacheObjects(cconfig)
-	if err != nil {
-		t.Fatal(err)
-	}
-	cobj := cobjects.(*cacheObjects)
-	GlobalServiceDoneCh <- struct{}{}
+
 	testCases := []struct {
 		bucketName     string
 		objectName     string
@@ -155,8 +148,8 @@ func TestCacheExclusion(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		cobj.exclude = []string{testCase.excludePattern}
-		if cobj.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
+		cobjects.exclude = []string{testCase.excludePattern}
+		if cobjects.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
 			t.Fatal("Cache exclusion test failed for case ", i)
 		}
 	}

cmd/dummy-handlers.go

@@ -72,12 +72,6 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 	w.(http.Flusher).Flush()
 }
 
-// GetBucketLifecycleHandler - GET bucket lifecycle, a dummy api
-func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
-	writeSuccessResponseHeadersOnly(w)
-	w.(http.Flusher).Flush()
-}
-
 // GetBucketReplicationHandler - GET bucket replication, a dummy api
 func (api objectAPIHandlers) GetBucketReplicationHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseHeadersOnly(w)

cmd/fs-v1-helpers.go

@@ -280,7 +280,7 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
 	}
 
 	// Stat to get the size of the file at path.
-	st, err := os.Stat(readPath)
+	st, err := fr.Stat()
 	if err != nil {
 		err = osErrToFSFileErr(err)
 		if err != errFileNotFound {
@@ -386,6 +386,26 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) {
 	return nil
 }
 
+// Renames source path to destination path, fails if the destination path
+// parents are not already created.
+func fsSimpleRenameFile(ctx context.Context, sourcePath, destPath string) error {
+	if err := checkPathLength(sourcePath); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	if err := checkPathLength(destPath); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+
+	if err := os.Rename(sourcePath, destPath); err != nil {
+		logger.LogIf(ctx, err)
+		return osErrToFSFileErr(err)
+	}
+
+	return nil
+}
+
 // Renames source path to destination path, creates all the
 // missing parents if they don't exist.
 func fsRenameFile(ctx context.Context, sourcePath, destPath string) error {

cmd/fs-v1-multipart.go

@@ -152,7 +152,7 @@ func (fs *FSObjects) ListMultipartUploads(ctx context.Context, bucket, object, k
 		return result, toObjectErr(err)
 	}
 
-	// S3 spec says uploaIDs should be sorted based on initiated time. ModTime of fs.json
+	// S3 spec says uploadIDs should be sorted based on initiated time. ModTime of fs.json
 	// is the creation time of the uploadID, hence we will use that.
 	var uploads []MultipartInfo
 	for _, uploadID := range uploadIDs {
@@ -326,7 +326,11 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 
 	partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag, data.ActualSize()))
 
-	if err = fsRenameFile(ctx, tmpPartPath, partPath); err != nil {
+	// Make sure not to create parent directories if they don't exist - the upload might have been aborted.
+	if err = fsSimpleRenameFile(ctx, tmpPartPath, partPath); err != nil {
+		if err == errFileNotFound || err == errFileAccessDenied {
+			return pi, InvalidUploadID{UploadID: uploadID}
+		}
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 

cmd/fs-v1.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/lock"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/mimedb"
@@ -1170,6 +1171,21 @@ func (fs *FSObjects) DeleteBucketPolicy(ctx context.Context, bucket string) erro
 	return removePolicyConfig(ctx, fs, bucket)
 }
 
+// SetBucketLifecycle sets lifecycle on bucket
+func (fs *FSObjects) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
+	return saveLifecycleConfig(ctx, fs, bucket, lifecycle)
+}
+
+// GetBucketLifecycle will get lifecycle on bucket
+func (fs *FSObjects) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
+	return getLifecycleConfig(fs, bucket)
+}
+
+// DeleteBucketLifecycle deletes all lifecycle on bucket
+func (fs *FSObjects) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
+	return removeLifecycleConfig(ctx, fs, bucket)
+}
+
 // ListObjectsV2 lists all blobs in bucket filtered by prefix
 func (fs *FSObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
 	marker := continuationToken

cmd/gateway-common.go

@@ -317,6 +317,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = BucketNotEmpty{}
 	case "NoSuchBucketPolicy":
 		err = BucketPolicyNotFound{}
+	case "NoSuchBucketLifecycle":
+		err = BucketLifecycleNotFound{}
 	case "InvalidBucketName":
 		err = BucketNameInvalid{Bucket: bucket}
 	case "InvalidPart":

cmd/gateway-main.go

@@ -269,6 +269,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Initialize policy system.
 	go globalPolicySys.Init(newObject)
 
+	// Create new lifecycle system
+	globalLifecycleSys = NewLifecycleSys()
+
 	// Create new notification system.
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 	if globalEtcdClient != nil && newObject.IsNotificationSupported() {

cmd/gateway-unsupported.go

@@ -20,6 +20,7 @@ import (
 	"context"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -81,6 +82,22 @@ func (a GatewayUnsupported) DeleteBucketPolicy(ctx context.Context, bucket strin
 	return NotImplemented{}
 }
 
+// SetBucketLifecycle sets lifecycle on bucket
+func (a GatewayUnsupported) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
+	logger.LogIf(ctx, NotImplemented{})
+	return NotImplemented{}
+}
+
+// GetBucketLifecycle will get lifecycle on bucket
+func (a GatewayUnsupported) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
+	return nil, NotImplemented{}
+}
+
+// DeleteBucketLifecycle deletes all lifecycle on bucket
+func (a GatewayUnsupported) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
+	return NotImplemented{}
+}
+
 // ReloadFormat - Not implemented stub.
 func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error {
 	return NotImplemented{}

cmd/gateway/azure/gateway-azure.go

@@ -111,7 +111,7 @@ EXAMPLES:
 
 	minio.RegisterGatewayCommand(cli.Command{
 		Name:               azureBackend,
-		Usage:              "Microsoft Azure Blob Storage.",
+		Usage:              "Microsoft Azure Blob Storage",
 		Action:             azureGatewayMain,
 		CustomHelpTemplate: azureGatewayTemplate,
 		HideHelpCommand:    true,

cmd/gateway/s3/gateway-s3.go

@@ -77,37 +77,37 @@ ENVIRONMENT VARIABLES:
 
 EXAMPLES:
   1. Start minio gateway server for AWS S3 backend.
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.HelpName}}
 
   2. Start minio gateway server for S3 backend on custom endpoint.
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
      {{.Prompt}} {{.HelpName}} https://play.min.io:9000
 
   3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint.
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"http://localhost:8000/"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}Q3AM3UQ867SPQQA43P2F
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_LOGGER_HTTP_ENDPOINT{{.AssignmentOperator}}"http://localhost:8000/"
      {{.Prompt}} {{.HelpName}} https://play.min.io:9000
 
   4. Start minio gateway server for AWS S3 backend with edge caching enabled.
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}"bucket1/*;*.png"
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}40
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}80
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXPIRY{{.AssignmentOperator}}40
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_MAXUSE{{.AssignmentOperator}}80
      {{.Prompt}} {{.HelpName}}
 
   4. Start minio gateway server for AWS S3 backend using AWS environment variables.
      NOTE: The access and secret key in this case will authenticate with MinIO instead
      of AWS and AWS envs will be used to authenticate to AWS S3.
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}aws_access_key
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}aws_secret_key
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}accesskey
-     {{.Prompt}} {{.EnvVarSetCommand}}(.*){{.AssignmentOperator}}secretkey
+     {{.Prompt}} {{.EnvVarSetCommand}} AWS_ACCESS_KEY_ID{{.AssignmentOperator}}aws_access_key
+     {{.Prompt}} {{.EnvVarSetCommand}} AWS_SECRET_ACCESS_KEY{{.AssignmentOperator}}aws_secret_key
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
+     {{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
      {{.Prompt}} {{.HelpName}}
 `
 

cmd/generic-handlers.go

@@ -473,7 +473,6 @@ var notimplementedBucketResourceNames = map[string]bool{
 	"acl":            true,
 	"cors":           true,
 	"inventory":      true,
-	"lifecycle":      true,
 	"logging":        true,
 	"metrics":        true,
 	"replication":    true,

cmd/globals.go

@@ -83,6 +83,11 @@ const (
 	// GlobalMultipartCleanupInterval - Cleanup interval when the stale multipart cleanup is initiated.
 	GlobalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
 
+	// GlobalServiceExecutionInterval - Executes the Lifecycle events.
+	GlobalServiceExecutionInterval = time.Hour * 24 // 24 hrs.
+
+	// Refresh interval to update in-memory bucket lifecycle cache.
+	globalRefreshBucketLifecycleInterval = 5 * time.Minute
 	// Refresh interval to update in-memory iam config cache.
 	globalRefreshIAMInterval = 5 * time.Minute
 
@@ -148,6 +153,8 @@ var (
 	globalPolicySys       *PolicySys
 	globalIAMSys          *IAMSys
 
+	globalLifecycleSys *LifecycleSys
+
 	// CA root certificates, a nil value means system certs pool will be used
 	globalRootCAs *x509.CertPool
 

cmd/handler-utils.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
-	xnet "github.com/minio/minio/pkg/net"
 )
 
 // Parses location constraint from the incoming reader.
@@ -387,15 +386,28 @@ func notFoundHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 // gets host name for current node
-func getHostName(r *http.Request) (hostName string, err error) {
-	var thisAddr *xnet.Host
-	hostName = r.Host
+func getHostName(r *http.Request) (hostName string) {
 	if globalIsDistXL {
-		thisAddr, err = xnet.ParseHost(GetLocalPeer(globalEndpoints))
-		if err != nil {
-			return
-		}
-		hostName = thisAddr.String()
+		hostName = GetLocalPeer(globalEndpoints)
+	} else {
+		hostName = r.Host
 	}
 	return
 }
+
+func isHTTPStatusOK(statusCode int) bool {
+	// List of success status.
+	var successStatus = []int{
+		http.StatusOK,
+		http.StatusCreated,
+		http.StatusAccepted,
+		http.StatusNoContent,
+		http.StatusPartialContent,
+	}
+	for _, okstatus := range successStatus {
+		if statusCode == okstatus {
+			return true
+		}
+	}
+	return false
+}

cmd/http-tracer.go

@@ -32,6 +32,8 @@ import (
 	trace "github.com/minio/minio/pkg/trace"
 )
 
+var traceBodyPlaceHolder = []byte("<BODY>")
+
 // recordRequest - records the first recLen bytes
 // of a given io.Reader
 type recordRequest struct {
@@ -41,10 +43,16 @@ type recordRequest struct {
 	logBody bool
 	// Internal recording buffer
 	buf bytes.Buffer
+	// request headers
+	headers http.Header
+	// total bytes read including header size
+	bytesRead int
 }
 
 func (r *recordRequest) Read(p []byte) (n int, err error) {
 	n, err = r.Reader.Read(p)
+	r.bytesRead += n
+
 	if r.logBody {
 		r.buf.Write(p[:n])
 	}
@@ -53,10 +61,22 @@ func (r *recordRequest) Read(p []byte) (n int, err error) {
 	}
 	return n, err
 }
+func (r *recordRequest) Size() int {
+	sz := r.bytesRead
+	for k, v := range r.headers {
+		sz += len(k) + len(v)
+	}
+	return sz
+}
 
 // Return the bytes that were recorded.
 func (r *recordRequest) Data() []byte {
-	return r.buf.Bytes()
+	// If body logging is enabled then we return the actual body
+	if r.logBody {
+		return r.buf.Bytes()
+	}
+	// ... otherwise we return <BODY> placeholder
+	return traceBodyPlaceHolder
 }
 
 // recordResponseWriter - records the first recLen bytes
@@ -73,13 +93,17 @@ type recordResponseWriter struct {
 	statusCode int
 	// Indicate if headers are written in the log
 	headersLogged bool
+	// number of bytes written
+	bytesWritten int
 }
 
 // Write the headers into the given buffer
-func writeHeaders(w io.Writer, statusCode int, headers http.Header) {
-	fmt.Fprintf(w, "%d %s\n", statusCode, http.StatusText(statusCode))
+func (r *recordResponseWriter) writeHeaders(w io.Writer, statusCode int, headers http.Header) {
+	n, _ := fmt.Fprintf(w, "%d %s\n", statusCode, http.StatusText(statusCode))
+	r.bytesWritten += n
 	for k, v := range headers {
-		fmt.Fprintf(w, "%s: %s\n", k, v[0])
+		n, _ := fmt.Fprintf(w, "%s: %s\n", k, v[0])
+		r.bytesWritten += n
 	}
 }
 
@@ -87,7 +111,7 @@ func writeHeaders(w io.Writer, statusCode int, headers http.Header) {
 func (r *recordResponseWriter) WriteHeader(i int) {
 	r.statusCode = i
 	if !r.headersLogged {
-		writeHeaders(&r.headers, i, r.ResponseWriter.Header())
+		r.writeHeaders(&r.headers, i, r.ResponseWriter.Header())
 		r.headersLogged = true
 	}
 	r.ResponseWriter.WriteHeader(i)
@@ -95,10 +119,11 @@ func (r *recordResponseWriter) WriteHeader(i int) {
 
 func (r *recordResponseWriter) Write(p []byte) (n int, err error) {
 	n, err = r.ResponseWriter.Write(p)
+	r.bytesWritten += n
 	if !r.headersLogged {
 		// We assume the response code to be '200 OK' when WriteHeader() is not called,
 		// that way following Golang HTTP response behavior.
-		writeHeaders(&r.headers, http.StatusOK, r.ResponseWriter.Header())
+		r.writeHeaders(&r.headers, http.StatusOK, r.ResponseWriter.Header())
 		r.headersLogged = true
 	}
 	if (r.statusCode != http.StatusOK && r.statusCode != http.StatusPartialContent && r.statusCode != 0) || r.logBody {
@@ -108,6 +133,10 @@ func (r *recordResponseWriter) Write(p []byte) (n int, err error) {
 	return n, err
 }
 
+func (r *recordResponseWriter) Size() int {
+	return r.bytesWritten
+}
+
 // Calls the underlying Flush.
 func (r *recordResponseWriter) Flush() {
 	r.ResponseWriter.(http.Flusher).Flush()
@@ -115,7 +144,13 @@ func (r *recordResponseWriter) Flush() {
 
 // Return response body.
 func (r *recordResponseWriter) Body() []byte {
-	return r.body.Bytes()
+	// If there was an error response or body logging is enabled
+	// then we return the body contents
+	if r.statusCode >= 400 || r.logBody {
+		return r.body.Bytes()
+	}
+	// ... otherwise we return the <BODY> place holder
+	return traceBodyPlaceHolder
 }
 
 // getOpName sanitizes the operation name for mc
@@ -136,14 +171,19 @@ func getOpName(name string) (op string) {
 
 // Trace gets trace of http request
 func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Request) trace.Info {
-
 	name := getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name())
 
-	bodyPlaceHolder := []byte("<BODY>")
-	var reqBodyRecorder *recordRequest
+	// Setup a http request body recorder
+	reqHeaders := cloneHeader(r.Header)
+	reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
+	reqHeaders.Set("Host", r.Host)
+	for _, enc := range r.TransferEncoding {
+		reqHeaders.Add("Transfer-Encoding", enc)
+	}
 
+	var reqBodyRecorder *recordRequest
 	t := trace.Info{FuncName: name}
-	reqBodyRecorder = &recordRequest{Reader: r.Body, logBody: logBody}
+	reqBodyRecorder = &recordRequest{Reader: r.Body, logBody: logBody, headers: reqHeaders}
 	r.Body = ioutil.NopCloser(reqBodyRecorder)
 	t.NodeName = r.Host
 	if globalIsDistXL {
@@ -154,41 +194,34 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 		t.NodeName = host
 	}
 
-	rq := trace.RequestInfo{Time: time.Now().UTC(), Method: r.Method, Path: r.URL.Path, RawQuery: r.URL.RawQuery, Client: r.RemoteAddr}
-	rq.Headers = cloneHeader(r.Header)
-	rq.Headers.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
-	rq.Headers.Set("Host", r.Host)
-	for _, enc := range r.TransferEncoding {
-		rq.Headers.Add("Transfer-Encoding", enc)
+	rq := trace.RequestInfo{
+		Time:     time.Now().UTC(),
+		Method:   r.Method,
+		Path:     r.URL.Path,
+		RawQuery: r.URL.RawQuery,
+		Client:   r.RemoteAddr,
+		Headers:  reqHeaders,
+		Body:     reqBodyRecorder.Data(),
 	}
-	if logBody {
-		// If body logging is disabled then we print <BODY> as a placeholder
-		// for the actual body.
-		rq.Body = reqBodyRecorder.Data()
 
-	} else {
-		rq.Body = bodyPlaceHolder
-	}
 	// Setup a http response body recorder
 	respBodyRecorder := &recordResponseWriter{ResponseWriter: w, logBody: logBody}
 	f(respBodyRecorder, r)
 
-	rs := trace.ResponseInfo{Time: time.Now().UTC()}
-	rs.Headers = cloneHeader(respBodyRecorder.Header())
-	rs.StatusCode = respBodyRecorder.statusCode
+	rs := trace.ResponseInfo{
+		Time:       time.Now().UTC(),
+		Headers:    cloneHeader(respBodyRecorder.Header()),
+		StatusCode: respBodyRecorder.statusCode,
+		Body:       respBodyRecorder.Body(),
+	}
+
 	if rs.StatusCode == 0 {
 		rs.StatusCode = http.StatusOK
 	}
-	bodyContents := respBodyRecorder.Body()
-	if bodyContents != nil {
-		rs.Body = bodyContents
-	}
-	if !logBody {
-		// If there was no error response and body logging is disabled
-		// then we print <BODY> as a placeholder for the actual body.
-		rs.Body = bodyPlaceHolder
-	}
+
 	t.ReqInfo = rq
 	t.RespInfo = rs
+
+	t.CallStats = trace.CallStats{Latency: rs.Time.Sub(rq.Time), InputBytes: reqBodyRecorder.Size(), OutputBytes: respBodyRecorder.Size()}
 	return t
 }

cmd/lifecycle.go

@@ -0,0 +1,191 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/xml"
+	"path"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
+)
+
+const (
+
+	// Disabled means the lifecycle rule is inactive
+	Disabled = "Disabled"
+)
+
+// LifecycleSys - Bucket lifecycle subsystem.
+type LifecycleSys struct {
+	sync.RWMutex
+	bucketLifecycleMap map[string]lifecycle.Lifecycle
+}
+
+// Set - sets lifecycle config to given bucket name.
+func (sys *LifecycleSys) Set(bucketName string, lifecycle lifecycle.Lifecycle) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	sys.bucketLifecycleMap[bucketName] = lifecycle
+}
+
+func saveLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string, bucketLifecycle *lifecycle.Lifecycle) error {
+	data, err := xml.Marshal(bucketLifecycle)
+	if err != nil {
+		return err
+	}
+
+	// Construct path to lifecycle.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
+	return saveConfig(ctx, objAPI, configFile, data)
+}
+
+// getLifecycleConfig - get lifecycle config for given bucket name.
+func getLifecycleConfig(objAPI ObjectLayer, bucketName string) (*lifecycle.Lifecycle, error) {
+	// Construct path to lifecycle.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
+	configData, err := readConfig(context.Background(), objAPI, configFile)
+	if err != nil {
+		if err == errConfigNotFound {
+			err = BucketLifecycleNotFound{Bucket: bucketName}
+		}
+		return nil, err
+	}
+
+	return lifecycle.ParseLifecycleConfig(bytes.NewReader(configData))
+}
+
+func removeLifecycleConfig(ctx context.Context, objAPI ObjectLayer, bucketName string) error {
+	// Construct path to lifecycle.xml for the given bucket.
+	configFile := path.Join(bucketConfigPrefix, bucketName, bucketLifecycleConfig)
+
+	if err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile); err != nil {
+		if _, ok := err.(ObjectNotFound); ok {
+			return BucketLifecycleNotFound{Bucket: bucketName}
+		}
+		return err
+	}
+	return nil
+}
+
+// NewLifecycleSys - creates new lifecycle system.
+func NewLifecycleSys() *LifecycleSys {
+	return &LifecycleSys{
+		bucketLifecycleMap: make(map[string]lifecycle.Lifecycle),
+	}
+}
+
+// Init - initializes lifecycle system from lifecycle.xml of all buckets.
+func (sys *LifecycleSys) Init(objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	defer func() {
+		// Refresh LifecycleSys in background.
+		go func() {
+			ticker := time.NewTicker(globalRefreshBucketLifecycleInterval)
+			defer ticker.Stop()
+			for {
+				select {
+				case <-GlobalServiceDoneCh:
+					return
+				case <-ticker.C:
+					sys.refresh(objAPI)
+				}
+			}
+		}()
+	}()
+
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Initializing lifecycle needs a retry mechanism for
+	// the following reasons:
+	//  - Read quorum is lost just after the initialization
+	//    of the object layer.
+	for range newRetryTimerSimple(doneCh) {
+		// Load LifecycleSys once during boot.
+		if err := sys.refresh(objAPI); err != nil {
+			if err == errDiskNotFound ||
+				strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
+				strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
+				logger.Info("Waiting for lifecycle subsystem to be initialized..")
+				continue
+			}
+			return err
+		}
+		break
+	}
+	return nil
+}
+
+// Refresh LifecycleSys.
+func (sys *LifecycleSys) refresh(objAPI ObjectLayer) error {
+	buckets, err := objAPI.ListBuckets(context.Background())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return err
+	}
+	sys.removeDeletedBuckets(buckets)
+	for _, bucket := range buckets {
+		config, err := objAPI.GetBucketLifecycle(context.Background(), bucket.Name)
+		if err != nil {
+			if _, ok := err.(BucketLifecycleNotFound); ok {
+				sys.Remove(bucket.Name)
+			}
+			continue
+		}
+
+		sys.Set(bucket.Name, *config)
+	}
+
+	return nil
+}
+
+// removeDeletedBuckets - to handle a corner case where we have cached the lifecycle for a deleted
+// bucket. i.e if we miss a delete-bucket notification we should delete the corresponding
+// bucket policy during sys.refresh()
+func (sys *LifecycleSys) removeDeletedBuckets(bucketInfos []BucketInfo) {
+	buckets := set.NewStringSet()
+	for _, info := range bucketInfos {
+		buckets.Add(info.Name)
+	}
+	sys.Lock()
+	defer sys.Unlock()
+
+	for bucket := range sys.bucketLifecycleMap {
+		if !buckets.Contains(bucket) {
+			delete(sys.bucketLifecycleMap, bucket)
+		}
+	}
+}
+
+// Remove - removes policy for given bucket name.
+func (sys *LifecycleSys) Remove(bucketName string) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	delete(sys.bucketLifecycleMap, bucketName)
+}

cmd/logger/logger.go

@@ -331,6 +331,7 @@ func logIf(ctx context.Context, err error) {
 		DeploymentID: req.DeploymentID,
 		Level:        ErrorLvl.String(),
 		RemoteHost:   req.RemoteHost,
+		Host:         req.Host,
 		RequestID:    req.RequestID,
 		UserAgent:    req.UserAgent,
 		Time:         time.Now().UTC().Format(time.RFC3339Nano),

cmd/logger/message/log/entry.go

@@ -43,6 +43,7 @@ type Entry struct {
 	Time         string `json:"time"`
 	API          *API   `json:"api,omitempty"`
 	RemoteHost   string `json:"remotehost,omitempty"`
+	Host         string `json:"host,omitempty"`
 	RequestID    string `json:"requestID,omitempty"`
 	UserAgent    string `json:"userAgent,omitempty"`
 	Message      string `json:"message,omitempty"`

cmd/logger/reqinfo.go

@@ -36,6 +36,7 @@ type KeyVal struct {
 // ReqInfo stores the request info.
 type ReqInfo struct {
 	RemoteHost   string   // Client Host/IP
+	Host         string   // Node Host/IP
 	UserAgent    string   // User Agent
 	DeploymentID string   // x-minio-deployment-id
 	RequestID    string   // x-amz-request-id

cmd/logger/target/console/console.go

@@ -89,6 +89,11 @@ func (c *Target) Send(e interface{}) error {
 		remoteHost = "\nRemoteHost: " + entry.RemoteHost
 	}
 
+	var host string
+	if entry.Host != "" {
+		host = "\nHost: " + entry.Host
+	}
+
 	var userAgent string
 	if entry.UserAgent != "" {
 		userAgent = "\nUserAgent: " + entry.UserAgent
@@ -99,8 +104,8 @@ func (c *Target) Send(e interface{}) error {
 	}
 
 	var msg = logger.ColorFgRed(logger.ColorBold(entry.Trace.Message))
-	var output = fmt.Sprintf("\n%s\n%s%s%s%s%s\nError: %s%s\n%s",
-		apiString, timeString, deploymentID, requestID, remoteHost, userAgent,
+	var output = fmt.Sprintf("\n%s\n%s%s%s%s%s%s\nError: %s%s\n%s",
+		apiString, timeString, deploymentID, requestID, remoteHost, host, userAgent,
 		msg, tagString, strings.Join(trace, "\n"))
 
 	fmt.Println(output)

cmd/naughty-disk_test.go

@@ -196,9 +196,9 @@ func (d *naughtyDisk) ReadAll(volume string, path string) (buf []byte, err error
 	return d.disk.ReadAll(volume, path)
 }
 
-func (d *naughtyDisk) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (d *naughtyDisk) VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.VerifyFile(volume, path, algo, sum, shardSize)
+	return d.disk.VerifyFile(volume, path, empty, algo, sum, shardSize)
 }

cmd/notification.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
@@ -498,6 +499,48 @@ func (sys *NotificationSys) RemoveBucketPolicy(ctx context.Context, bucketName s
 	}()
 }
 
+// SetBucketLifecycle - calls SetBucketLifecycle on all peers.
+func (sys *NotificationSys) SetBucketLifecycle(ctx context.Context, bucketName string, bucketLifecycle *lifecycle.Lifecycle) {
+	go func() {
+		var wg sync.WaitGroup
+		for _, client := range sys.peerClients {
+			if client == nil {
+				continue
+			}
+			wg.Add(1)
+			go func(client *peerRESTClient) {
+				defer wg.Done()
+				if err := client.SetBucketLifecycle(bucketName, bucketLifecycle); err != nil {
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
+					logger.LogIf(ctx, err)
+				}
+			}(client)
+		}
+		wg.Wait()
+	}()
+}
+
+// RemoveBucketLifecycle - calls RemoveLifecycle on all peers.
+func (sys *NotificationSys) RemoveBucketLifecycle(ctx context.Context, bucketName string) {
+	go func() {
+		var wg sync.WaitGroup
+		for _, client := range sys.peerClients {
+			if client == nil {
+				continue
+			}
+			wg.Add(1)
+			go func(client *peerRESTClient) {
+				defer wg.Done()
+				if err := client.RemoveBucketLifecycle(bucketName); err != nil {
+					logger.GetReqInfo(ctx).AppendTags("remotePeer", client.host.Name)
+					logger.LogIf(ctx, err)
+				}
+			}(client)
+		}
+		wg.Wait()
+	}()
+}
+
 // PutBucketNotification - calls PutBucketNotification RPC call on all peers.
 func (sys *NotificationSys) PutBucketNotification(ctx context.Context, bucketName string, rulesMap event.RulesMap) {
 	go func() {

cmd/object-api-errors.go

@@ -254,6 +254,13 @@ func (e BucketPolicyNotFound) Error() string {
 	return "No bucket policy found for bucket: " + e.Bucket
 }
 
+// BucketLifecycleNotFound - no bucket lifecycle found.
+type BucketLifecycleNotFound GenericError
+
+func (e BucketLifecycleNotFound) Error() string {
+	return "No bucket life cycle found for bucket : " + e.Bucket
+}
+
 /// Bucket related errors.
 
 // BucketNameInvalid - bucketname provided is invalid.
@@ -269,11 +276,27 @@ func (e BucketNameInvalid) Error() string {
 // ObjectNameInvalid - object name provided is invalid.
 type ObjectNameInvalid GenericError
 
+// ObjectNameTooLong - object name too long.
+type ObjectNameTooLong GenericError
+
+// ObjectNamePrefixAsSlash - object name has a slash as prefix.
+type ObjectNamePrefixAsSlash GenericError
+
 // Return string an error formatted as the given text.
 func (e ObjectNameInvalid) Error() string {
 	return "Object name invalid: " + e.Bucket + "#" + e.Object
 }
 
+// Return string an error formatted as the given text.
+func (e ObjectNameTooLong) Error() string {
+	return "Object name too long: " + e.Bucket + "#" + e.Object
+}
+
+// Return string an error formatted as the given text.
+func (e ObjectNamePrefixAsSlash) Error() string {
+	return "Object name contains forward slash as pefix: " + e.Bucket + "#" + e.Object
+}
+
 // AllAccessDisabled All access to this object has been disabled
 type AllAccessDisabled GenericError
 

cmd/object-api-input-checks.go

@@ -166,18 +166,18 @@ func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer
 	if err := checkBucketExist(ctx, bucket, obj); err != nil {
 		return err
 	}
+
+	if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil {
+		return err
+	}
 	// Validates object name validity after bucket exists.
 	if !IsValidObjectName(object) {
-		logger.LogIf(ctx, ObjectNameInvalid{
-			Bucket: bucket,
-			Object: object,
-		})
-
 		return ObjectNameInvalid{
 			Bucket: bucket,
 			Object: object,
 		}
 	}
+
 	return nil
 }
 
@@ -192,8 +192,10 @@ func checkPutObjectArgs(ctx context.Context, bucket, object string, obj ObjectLa
 		return err
 	}
 
+	if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil {
+		return err
+	}
 	if len(object) == 0 ||
-		hasPrefix(object, slashSeparator) ||
 		(hasSuffix(object, slashSeparator) && size != 0) ||
 		!IsValidObjectPrefix(object) {
 		return ObjectNameInvalid{

cmd/object-api-interface.go

@@ -22,6 +22,7 @@ import (
 	"net/http"
 
 	"github.com/minio/minio-go/v6/pkg/encrypt"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 )
@@ -107,4 +108,9 @@ type ObjectLayer interface {
 
 	// Compression support check.
 	IsCompressionSupported() bool
+
+	// Lifecycle operations
+	SetBucketLifecycle(context.Context, string, *lifecycle.Lifecycle) error
+	GetBucketLifecycle(context.Context, string) (*lifecycle.Lifecycle, error)
+	DeleteBucketLifecycle(context.Context, string) error
 }

cmd/object-api-utils.go

@@ -136,7 +136,7 @@ func IsValidObjectName(object string) bool {
 	if len(object) == 0 {
 		return false
 	}
-	if hasSuffix(object, slashSeparator) || hasPrefix(object, slashSeparator) {
+	if hasSuffix(object, slashSeparator) {
 		return false
 	}
 	return IsValidObjectPrefix(object)
@@ -148,9 +148,6 @@ func IsValidObjectPrefix(object string) bool {
 	if hasBadPathComponent(object) {
 		return false
 	}
-	if len(object) > 1024 {
-		return false
-	}
 	if !utf8.ValidString(object) {
 		return false
 	}
@@ -161,6 +158,25 @@ func IsValidObjectPrefix(object string) bool {
 	return true
 }
 
+// checkObjectNameForLengthAndSlash -check for the validity of object name length and prefis as slash
+func checkObjectNameForLengthAndSlash(bucket, object string) error {
+	// Check for the length of object name
+	if len(object) > 1024 {
+		return ObjectNameTooLong{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+	// Check for slash as prefix in object name
+	if hasPrefix(object, slashSeparator) {
+		return ObjectNamePrefixAsSlash{
+			Bucket: bucket,
+			Object: object,
+		}
+	}
+	return nil
+}
+
 // Slash separator.
 const slashSeparator = "/"
 

cmd/object-api-utils_test.go

@@ -113,7 +113,6 @@ func TestIsValidObjectName(t *testing.T) {
 		// passing invalid object names.
 		{"", false},
 		{"a/b/c/", false},
-		{"/a/b/c", false},
 		{"../../etc", false},
 		{"../../", false},
 		{"/../../etc", false},

cmd/object-handlers.go

@@ -2280,6 +2280,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		w.Write(encodedErrorResponse)
 		w.(http.Flusher).Flush()
 	}
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
 	w = &whiteSpaceWriter{ResponseWriter: w, Flusher: w.(http.Flusher)}
 	completeDoneCh := sendWhiteSpace(ctx, w)
 	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, opts)

cmd/os-reliable.go

@@ -156,11 +156,11 @@ func renameAll(srcFilePath, dstFilePath string) (err error) {
 // Reliably retries os.RenameAll if for some reason os.RenameAll returns
 // syscall.ENOENT (parent does not exist).
 func reliableRename(srcFilePath, dstFilePath string) (err error) {
+	if err = reliableMkdirAll(path.Dir(dstFilePath), 0777); err != nil {
+		return err
+	}
 	i := 0
 	for {
-		if err = reliableMkdirAll(path.Dir(dstFilePath), 0777); err != nil {
-			return err
-		}
 		// After a successful parent directory create attempt a renameAll.
 		if err = os.Rename(srcFilePath, dstFilePath); err != nil {
 			// Retry only for the first retryable error.

cmd/peer-rest-client.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
@@ -329,6 +330,37 @@ func (client *peerRESTClient) SetBucketPolicy(bucket string, bucketPolicy *polic
 	return nil
 }
 
+// RemoveBucketLifecycle - Remove bucket lifecycle configuration on the peer node
+func (client *peerRESTClient) RemoveBucketLifecycle(bucket string) error {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+	respBody, err := client.call(peerRESTMethodBucketLifecycleRemove, values, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
+// SetBucketLifecycle - Set bucket lifecycle configuration on the peer node
+func (client *peerRESTClient) SetBucketLifecycle(bucket string, bucketLifecycle *lifecycle.Lifecycle) error {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+
+	var reader bytes.Buffer
+	err := gob.NewEncoder(&reader).Encode(bucketLifecycle)
+	if err != nil {
+		return err
+	}
+
+	respBody, err := client.call(peerRESTMethodBucketLifecycleSet, values, &reader, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}
+
 // PutBucketNotification - Put bucket notification on the peer node.
 func (client *peerRESTClient) PutBucketNotification(bucket string, rulesMap event.RulesMap) error {
 	values := make(url.Values)
@@ -435,9 +467,10 @@ func (client *peerRESTClient) BackgroundHealStatus() (madmin.BgHealState, error)
 	return state, err
 }
 
-func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh chan struct{}, trcAll bool) {
+func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh chan struct{}, trcAll, trcErr bool) {
 	values := make(url.Values)
 	values.Set(peerRESTTraceAll, strconv.FormatBool(trcAll))
+	values.Set(peerRESTTraceErr, strconv.FormatBool(trcErr))
 
 	// To cancel the REST request in case doneCh gets closed.
 	ctx, cancel := context.WithCancel(context.Background())
@@ -475,10 +508,10 @@ func (client *peerRESTClient) doTrace(traceCh chan interface{}, doneCh chan stru
 }
 
 // Trace - send http trace request to peer nodes
-func (client *peerRESTClient) Trace(traceCh chan interface{}, doneCh chan struct{}, trcAll bool) {
+func (client *peerRESTClient) Trace(traceCh chan interface{}, doneCh chan struct{}, trcAll, trcErr bool) {
 	go func() {
 		for {
-			client.doTrace(traceCh, doneCh, trcAll)
+			client.doTrace(traceCh, doneCh, trcAll, trcErr)
 			select {
 			case <-doneCh:
 				return

cmd/peer-rest-common.go

@@ -43,6 +43,8 @@ const (
 	peerRESTMethodTargetExists             = "targetexists"
 	peerRESTMethodSendEvent                = "sendevent"
 	peerRESTMethodTrace                    = "trace"
+	peerRESTMethodBucketLifecycleSet       = "setbucketlifecycle"
+	peerRESTMethodBucketLifecycleRemove    = "removebucketlifecycle"
 )
 
 const (
@@ -54,4 +56,5 @@ const (
 	peerRESTProfiler = "profiler"
 	peerRESTDryRun   = "dry-run"
 	peerRESTTraceAll = "all"
+	peerRESTTraceErr = "err"
 )

cmd/peer-rest-server.go

@@ -31,6 +31,7 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/lifecycle"
 	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/policy"
 	trace "github.com/minio/minio/pkg/trace"
@@ -469,6 +470,47 @@ func (s *peerRESTServer) SetBucketPolicyHandler(w http.ResponseWriter, r *http.R
 	w.(http.Flusher).Flush()
 }
 
+// RemoveBucketLifecycleHandler - Remove bucket lifecycle.
+func (s *peerRESTServer) RemoveBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+
+	globalLifecycleSys.Remove(bucketName)
+	w.(http.Flusher).Flush()
+}
+
+// SetBucketLifecycleHandler - Set bucket lifecycle.
+func (s *peerRESTServer) SetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	if bucketName == "" {
+		s.writeErrorResponse(w, errors.New("Bucket name is missing"))
+		return
+	}
+	var lifecycleData lifecycle.Lifecycle
+	if r.ContentLength < 0 {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	err := gob.NewDecoder(r.Body).Decode(&lifecycleData)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	globalLifecycleSys.Set(bucketName, lifecycleData)
+	w.(http.Flusher).Flush()
+}
+
 type remoteTargetExistsResp struct {
 	Exists bool
 }
@@ -675,17 +717,23 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	trcAll := r.URL.Query().Get(peerRESTTraceAll) == "true"
+	trcErr := r.URL.Query().Get(peerRESTTraceErr) == "true"
 
 	w.Header().Set(xhttp.Connection, "close")
 	w.WriteHeader(http.StatusOK)
 	w.(http.Flusher).Flush()
 
 	filter := func(entry interface{}) bool {
+		trcInfo := entry.(trace.Info)
+
+		if trcErr && isHTTPStatusOK(trcInfo.RespInfo.StatusCode) {
+			return false
+		}
 		if trcAll {
 			return true
 		}
-		trcInfo := entry.(trace.Info)
 		return !strings.HasPrefix(trcInfo.ReqInfo.Path, minioReservedBucketPath)
+
 	}
 
 	doneCh := make(chan struct{})
@@ -768,6 +816,8 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketNotificationListen).HandlerFunc(httpTraceHdrs(server.ListenBucketNotificationHandler)).Queries(restQueries(peerRESTBucket)...)
 
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodReloadFormat).HandlerFunc(httpTraceHdrs(server.ReloadFormatHandler)).Queries(restQueries(peerRESTDryRun)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketLifecycleSet).HandlerFunc(httpTraceHdrs(server.SetBucketLifecycleHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBucketLifecycleRemove).HandlerFunc(httpTraceHdrs(server.RemoveBucketLifecycleHandler)).Queries(restQueries(peerRESTBucket)...)
 
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
 	subrouter.Methods(http.MethodPost).Path("/" + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)

cmd/posix.go

@@ -1526,7 +1526,7 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 	return nil
 }
 
-func (s *posix) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
+func (s *posix) VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
 	defer func() {
 		if err == errFaultyDisk {
 			atomic.AddInt32(&s.ioErrCount, 1)
@@ -1546,7 +1546,7 @@ func (s *posix) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte
 		return err
 	}
 	// Stat a volume entry.
-	_, err = os.Stat((volumeDir))
+	_, err = os.Stat(volumeDir)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return errVolumeNotFound
@@ -1556,12 +1556,12 @@ func (s *posix) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte
 
 	// Validate effective path length before reading.
 	filePath := pathJoin(volumeDir, path)
-	if err = checkPathLength((filePath)); err != nil {
+	if err = checkPathLength(filePath); err != nil {
 		return err
 	}
 
 	// Open the file for reading.
-	file, err := os.Open((filePath))
+	file, err := os.Open(filePath)
 	if err != nil {
 		switch {
 		case os.IsNotExist(err):
@@ -1601,6 +1601,11 @@ func (s *posix) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte
 	if err != nil {
 		return err
 	}
+
+	if empty && fi.Size() != 0 || !empty && fi.Size() == 0 {
+		return errFileUnexpectedSize
+	}
+
 	size := fi.Size()
 	for {
 		if size == 0 {

cmd/posix_test.go

@@ -1797,7 +1797,7 @@ func TestPosixVerifyFile(t *testing.T) {
 	if err := posixStorage.WriteAll(volName, fileName, bytes.NewBuffer(data)); err != nil {
 		t.Fatal(err)
 	}
-	if err := posixStorage.VerifyFile(volName, fileName, algo, hashBytes, 0); err != nil {
+	if err := posixStorage.VerifyFile(volName, fileName, false, algo, hashBytes, 0); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1805,7 +1805,7 @@ func TestPosixVerifyFile(t *testing.T) {
 	if err := posixStorage.AppendFile(volName, fileName, []byte("a")); err != nil {
 		t.Fatal(err)
 	}
-	if err := posixStorage.VerifyFile(volName, fileName, algo, hashBytes, 0); err == nil {
+	if err := posixStorage.VerifyFile(volName, fileName, false, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
@@ -1830,12 +1830,10 @@ func TestPosixVerifyFile(t *testing.T) {
 		if err == io.EOF {
 			break
 		}
-		if err != nil {
-			t.Fatal(err)
-		}
+		t.Fatal(err)
 	}
 	w.Close()
-	if err := posixStorage.VerifyFile(volName, fileName, algo, nil, shardSize); err != nil {
+	if err := posixStorage.VerifyFile(volName, fileName, false, algo, nil, shardSize); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1849,7 +1847,7 @@ func TestPosixVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	f.Close()
-	if err := posixStorage.VerifyFile(volName, fileName, algo, nil, shardSize); err == nil {
+	if err := posixStorage.VerifyFile(volName, fileName, false, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 }

cmd/postpolicyform.go

@@ -109,7 +109,7 @@ type PostPolicyForm struct {
 	}
 }
 
-// parsePostPolicyForm - Parse JSON policy string into typed POostPolicyForm structure.
+// parsePostPolicyForm - Parse JSON policy string into typed PostPolicyForm structure.
 func parsePostPolicyForm(policy string) (ppf PostPolicyForm, e error) {
 	// Convert po into interfaces and
 	// perform strict type conversion using reflection.
@@ -256,6 +256,7 @@ func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) erro
 				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
 			}
 			// Check if current policy condition is satisfied
+			condPassed = checkPolicyCond(op, formValues.Get(formCanonicalName), v.Value)
 			if !condPassed {
 				return fmt.Errorf("Invalid according to Policy: Policy Condition failed")
 			}

cmd/server-main.go

@@ -365,6 +365,14 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(err, "Unable to initialize policy system")
 	}
 
+	// Create new lifecycle system.
+	globalLifecycleSys = NewLifecycleSys()
+
+	// Initialize lifecycle system.
+	if err = globalLifecycleSys.Init(newObject); err != nil {
+		logger.Fatal(err, "Unable to initialize lifecycle system")
+	}
+
 	// Create new notification system.
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 

cmd/server_test.go

@@ -1177,7 +1177,7 @@ func (s *TestSuiteCommon) TestPutObject(c *check) {
 	c.Assert(response.StatusCode, http.StatusOK)
 	c.Assert(response.ContentLength, int64(len([]byte("hello world"))))
 	var buffer2 bytes.Buffer
-	// retrive the contents of response body.
+	// retrieve the contents of response body.
 	n, err := io.Copy(&buffer2, response.Body)
 	c.Assert(err, nil)
 	c.Assert(n, int64(len([]byte("hello world"))))
@@ -1354,7 +1354,37 @@ func (s *TestSuiteCommon) TestPutObjectLongName(c *check) {
 	response, err = client.Do(request)
 	c.Assert(err, nil)
 	c.Assert(response.StatusCode, http.StatusOK)
-	// make long object name.
+
+	//make long object name.
+	longObjName = fmt.Sprintf("%0255d/%0255d/%0255d/%0255d/%0255d", 1, 1, 1, 1, 1)
+	if IsDocker() || IsKubernetes() {
+		longObjName = fmt.Sprintf("%0242d/%0242d/%0242d/%0242d/%0242d", 1, 1, 1, 1, 1)
+	}
+	// create new HTTP request to insert the object.
+	buffer = bytes.NewReader([]byte("hello world"))
+	request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, longObjName),
+		int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer)
+	c.Assert(err, nil)
+	// execute the HTTP request.
+	response, err = client.Do(request)
+	c.Assert(err, nil)
+	c.Assert(response.StatusCode, http.StatusBadRequest)
+	verifyError(c, response, "KeyTooLongError", "Your key is too long", http.StatusBadRequest)
+
+	// make object name with prefix as slash
+	longObjName = fmt.Sprintf("/%0255d/%0255d", 1, 1)
+	buffer = bytes.NewReader([]byte("hello world"))
+	// create new HTTP request to insert the object.
+	request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, longObjName),
+		int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer)
+	c.Assert(err, nil)
+	// execute the HTTP request.
+	response, err = client.Do(request)
+	c.Assert(err, nil)
+	c.Assert(response.StatusCode, http.StatusBadRequest)
+	verifyError(c, response, "XMinioInvalidObjectName", "Object name contains a leading slash.", http.StatusBadRequest)
+
+	//make object name as unsuported
 	longObjName = fmt.Sprintf("%0256d", 1)
 	buffer = bytes.NewReader([]byte("hello world"))
 	request, err = newTestSignedRequest("PUT", getPutObjectURL(s.endPoint, bucketName, longObjName),
@@ -1431,7 +1461,7 @@ func (s *TestSuiteCommon) TestHeadOnObjectLastModified(c *check) {
 	// verify the status of the HTTP response.
 	c.Assert(response.StatusCode, http.StatusOK)
 
-	// retrive the info of last modification time of the object from the response header.
+	// retrieve the info of last modification time of the object from the response header.
 	lastModified := response.Header.Get("Last-Modified")
 	// Parse it into time.Time structure.
 	t, err := time.Parse(http.TimeFormat, lastModified)

cmd/storage-errors.go

@@ -72,6 +72,9 @@ var errVolumeAccessDenied = errors.New("volume access denied")
 // errFileAccessDenied - cannot access file, insufficient permissions.
 var errFileAccessDenied = errors.New("file access denied")
 
+// errFileUnexpectedSize - file has an unexpected size
+var errFileUnexpectedSize = errors.New("file has unexpected size")
+
 // errFileParentIsFile - cannot have overlapping objects, parent is already a file.
 var errFileParentIsFile = errors.New("parent is a file")
 

cmd/storage-interface.go

@@ -52,7 +52,7 @@ type StorageAPI interface {
 	StatFile(volume string, path string) (file FileInfo, err error)
 	DeleteFile(volume string, path string) (err error)
 	DeleteFileBulk(volume string, paths []string) (errs []error, err error)
-	VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) error
+	VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) error
 
 	// Write all data, syncs the data to disk.
 	WriteAll(volume string, path string, reader io.Reader) (err error)

cmd/storage-rest-client.go

@@ -67,6 +67,8 @@ func toStorageErr(err error) error {
 		return io.EOF
 	case io.ErrUnexpectedEOF.Error():
 		return io.ErrUnexpectedEOF
+	case errFileUnexpectedSize.Error():
+		return errFileUnexpectedSize
 	case errUnexpected.Error():
 		return errUnexpected
 	case errDiskFull.Error():
@@ -434,11 +436,12 @@ func (client *storageRESTClient) getInstanceID() (err error) {
 	return nil
 }
 
-func (client *storageRESTClient) VerifyFile(volume, path string, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (client *storageRESTClient) VerifyFile(volume, path string, empty bool, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
 	values.Set(storageRESTBitrotAlgo, algo.String())
+	values.Set(storageRESTEmpty, strconv.FormatBool(empty))
 	values.Set(storageRESTLength, strconv.Itoa(int(shardSize)))
 	if len(sum) != 0 {
 		values.Set(storageRESTBitrotHash, hex.EncodeToString(sum))

cmd/storage-rest-common.go

@@ -16,7 +16,7 @@
 
 package cmd
 
-const storageRESTVersion = "v7"
+const storageRESTVersion = "v8"
 const storageRESTPath = minioReservedBucketPath + "/storage/" + storageRESTVersion + "/"
 
 const (
@@ -52,6 +52,7 @@ const (
 	storageRESTDstPath    = "destination-path"
 	storageRESTOffset     = "offset"
 	storageRESTLength     = "length"
+	storageRESTEmpty      = "empty"
 	storageRESTCount      = "count"
 	storageRESTMarkerPath = "marker"
 	storageRESTLeafFile   = "leaf-file"

cmd/storage-rest-server.go

@@ -520,6 +520,11 @@ func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
 	filePath := vars[storageRESTFilePath]
+	empty, err := strconv.ParseBool(vars[storageRESTEmpty])
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
 	shardSize, err := strconv.Atoi(vars[storageRESTLength])
 	if err != nil {
 		s.writeErrorResponse(w, err)
@@ -540,8 +545,9 @@ func (s *storageRESTServer) VerifyFile(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	algo := BitrotAlgorithmFromString(algoStr)
+	w.Header().Set(xhttp.ContentType, "text/event-stream")
 	doneCh := sendWhiteSpaceVerifyFile(w)
-	err = s.storage.VerifyFile(volume, filePath, algo, hash, int64(shardSize))
+	err = s.storage.VerifyFile(volume, filePath, empty, algo, hash, int64(shardSize))
 	<-doneCh
 	gob.NewEncoder(w).Encode(VerifyFileResp{err})
 }

cmd/test-utils_test.go

@@ -365,6 +365,9 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
 	globalNotificationSys = NewNotificationSys(globalServerConfig, testServer.Disks)
 	globalNotificationSys.Init(objLayer)
 
+	globalLifecycleSys = NewLifecycleSys()
+	globalLifecycleSys.Init(objLayer)
+
 	return testServer
 }
 

cmd/utils.go

@@ -429,6 +429,7 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 		DeploymentID: globalDeploymentID,
 		RequestID:    w.Header().Get(xhttp.AmzRequestID),
 		RemoteHost:   handlers.GetSourceIP(r),
+		Host:         getHostName(r),
 		UserAgent:    r.UserAgent(),
 		API:          api,
 		BucketName:   bucket,

cmd/web-handler-context.go

@@ -0,0 +1,245 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/handlers"
+)
+
+const (
+	kmBucket   = "BucketName"
+	kmObject   = "ObjectName"
+	kmObjects  = "Objects"
+	kmPrefix   = "Prefix"
+	kmMarker   = "Marker"
+	kmUsername = "UserName"
+	kmHostname = "HostName"
+	kmPolicy   = "Policy"
+)
+
+// KeyValueMap extends builtin map to support setting and getting
+// select fields like BucketName, ObjectName, Prefix, etc.
+type KeyValueMap map[string]string
+
+// Bucket returns the BucketName
+func (km KeyValueMap) Bucket() string {
+	return km[kmBucket]
+}
+
+// Object returns the ObjectName
+func (km KeyValueMap) Object() string {
+	return km[kmObject]
+}
+
+// Prefix returns the Prefix
+func (km KeyValueMap) Prefix() string {
+	return km[kmPrefix]
+}
+
+// Username returns the Username
+func (km KeyValueMap) Username() string {
+	return km[kmUsername]
+}
+
+// Hostname returns the Hostname
+func (km KeyValueMap) Hostname() string {
+	return km[kmHostname]
+}
+
+// Policy returns the Policy
+func (km KeyValueMap) Policy() string {
+	return km[kmPolicy]
+}
+
+// Objects returns the Objects
+func (km KeyValueMap) Objects() []string {
+	var objects []string
+	_ = json.Unmarshal([]byte(km[kmObjects]), &objects)
+	return objects
+}
+
+// SetBucket sets the given bucket to the KeyValueMap
+func (km *KeyValueMap) SetBucket(bucket string) {
+	(*km)[kmBucket] = bucket
+}
+
+// SetPrefix sets the given prefix to the KeyValueMap
+func (km *KeyValueMap) SetPrefix(prefix string) {
+	(*km)[kmPrefix] = prefix
+}
+
+// SetObject sets the given object to the KeyValueMap
+func (km *KeyValueMap) SetObject(object string) {
+	(*km)[kmObject] = object
+}
+
+// SetMarker sets the given marker to the KeyValueMap
+func (km *KeyValueMap) SetMarker(marker string) {
+	(*km)[kmMarker] = marker
+}
+
+// SetPolicy sets the given policy to the KeyValueMap
+func (km *KeyValueMap) SetPolicy(policy string) {
+	(*km)[kmPolicy] = policy
+}
+
+// SetExpiry sets the expiry to the KeyValueMap
+func (km *KeyValueMap) SetExpiry(expiry int64) {
+	(*km)[kmPolicy] = fmt.Sprintf("%d", expiry)
+}
+
+// SetObjects sets the list of objects to the KeyValueMap
+func (km *KeyValueMap) SetObjects(objects []string) {
+	objsVal, err := json.Marshal(objects)
+	if err != nil {
+		// NB this can only happen when we can't marshal a Go
+		// slice to its json representation.
+		objsVal = []byte("[]")
+	}
+	(*km)[kmObjects] = string(objsVal)
+}
+
+// SetUsername sets the username to the KeyValueMap
+func (km *KeyValueMap) SetUsername(username string) {
+	(*km)[kmUsername] = username
+}
+
+// SetHostname sets the hostname to the KeyValueMap
+func (km *KeyValueMap) SetHostname(hostname string) {
+	(*km)[kmHostname] = hostname
+}
+
+// ToKeyValuer interface wraps ToKeyValue method that allows types to
+// marshal their values as a map of structure member names to their
+// values, as strings
+type ToKeyValuer interface {
+	ToKeyValue() KeyValueMap
+}
+
+// ToKeyValue implementation for WebGenericArgs
+func (args *WebGenericArgs) ToKeyValue() KeyValueMap {
+	return KeyValueMap{}
+}
+
+// ToKeyValue implementation for MakeBucketArgs
+func (args *MakeBucketArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetBucket(args.BucketName)
+	return km
+}
+
+// ToKeyValue implementation for RemoveBucketArgs
+func (args *RemoveBucketArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetBucket(args.BucketName)
+	return km
+}
+
+// ToKeyValue implementation for ListObjectsArgs
+func (args *ListObjectsArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetBucket(args.BucketName)
+	km.SetPrefix(args.Prefix)
+	km.SetMarker(args.Marker)
+	return km
+}
+
+// ToKeyValue implementation for RemoveObjectArgs
+func (args *RemoveObjectArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetBucket(args.BucketName)
+	km.SetObjects(args.Objects)
+	return km
+}
+
+// ToKeyValue implementation for LoginArgs
+func (args *LoginArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetUsername(args.Username)
+	return km
+}
+
+// ToKeyValue implementation for GetBucketPolicyArgs
+func (args *GetBucketPolicyArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetBucket(args.BucketName)
+	km.SetPrefix(args.Prefix)
+	return km
+}
+
+// ToKeyValue implementation for ListAllBucketPoliciesArgs
+func (args *ListAllBucketPoliciesArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetBucket(args.BucketName)
+	return km
+}
+
+// ToKeyValue implementation for SetBucketPolicyWebArgs
+func (args *SetBucketPolicyWebArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetBucket(args.BucketName)
+	km.SetPrefix(args.Prefix)
+	km.SetPolicy(args.Policy)
+	return km
+}
+
+// ToKeyValue implementation for SetAuthArgs
+// SetAuthArgs doesn't implement the ToKeyValue interface that will be
+// used by logger subsystem down the line, to avoid leaking
+// credentials to an external log target
+func (args *SetAuthArgs) ToKeyValue() KeyValueMap {
+	return KeyValueMap{}
+}
+
+// ToKeyValue implementation for PresignedGetArgs
+func (args *PresignedGetArgs) ToKeyValue() KeyValueMap {
+	km := KeyValueMap{}
+	km.SetHostname(args.HostName)
+	km.SetBucket(args.BucketName)
+	km.SetObject(args.ObjectName)
+	km.SetExpiry(args.Expiry)
+	return km
+}
+
+// newWebContext creates a context with ReqInfo values from the given
+// http request and api name.
+func newWebContext(r *http.Request, args ToKeyValuer, api string) context.Context {
+	argsMap := args.ToKeyValue()
+	bucket := argsMap.Bucket()
+	object := argsMap.Object()
+	prefix := argsMap.Prefix()
+
+	if prefix != "" {
+		object = prefix
+	}
+	reqInfo := &logger.ReqInfo{
+		DeploymentID: globalDeploymentID,
+		RemoteHost:   handlers.GetSourceIP(r),
+		Host:         getHostName(r),
+		UserAgent:    r.UserAgent(),
+		API:          api,
+		BucketName:   bucket,
+		ObjectName:   object,
+	}
+	return logger.SetReqInfo(context.Background(), reqInfo)
+}

cmd/web-handler-context_test.go

@@ -0,0 +1,111 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"net/http"
+	"testing"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+func TestKeyValueMap(t *testing.T) {
+	bucket := "bucket"
+	object := "object"
+	prefix := "prefix"
+	username := "username"
+	policy := "policy"
+	host := "min.io"
+	objects := []string{object, object}
+
+	km := KeyValueMap{}
+	km.SetBucket(bucket)
+	km.SetPrefix(prefix)
+	km.SetUsername(username)
+	km.SetHostname(host)
+	km.SetObject(object)
+	km.SetObjects(objects)
+	km.SetPolicy(policy)
+
+	if got := km.Bucket(); got != bucket {
+		t.Errorf("Expected %s but got %s", bucket, got)
+	}
+
+	if got := km.Object(); got != object {
+		t.Errorf("Expected %s but got %s", object, got)
+	}
+
+	areEqualObjects := func(as, bs []string) bool {
+		if len(as) != len(bs) {
+			return false
+		}
+
+		for i, a := range as {
+			b := bs[i]
+			if a != b {
+				return false
+			}
+		}
+		return true
+	}
+
+	if got := km.Objects(); !areEqualObjects(got, objects) {
+		t.Errorf("Expected %s but got %s", objects, got)
+	}
+
+	if got := km.Policy(); got != policy {
+		t.Errorf("Expected %s but got %s", policy, got)
+	}
+
+	if got := km.Prefix(); got != prefix {
+		t.Errorf("Expected %s but got %s", prefix, got)
+	}
+
+	if got := km.Username(); got != username {
+		t.Errorf("Expected %s but got %s", username, got)
+	}
+
+	if got := km.Hostname(); got != host {
+		t.Errorf("Expected %s but got %s", host, got)
+	}
+}
+
+func TestNewWebContext(t *testing.T) {
+	api := "Test API"
+	args := ListObjectsArgs{
+		BucketName: "bucket",
+		Prefix:     "prefix",
+		Marker:     "marker",
+	}
+
+	req, err := http.NewRequest(http.MethodPost, "http://min.io", bytes.NewReader([]byte("nothing")))
+	if err != nil {
+		t.Fatal("Unexpected failure while creating a test request")
+	}
+
+	ctx := newWebContext(req, &args, api)
+	reqInfo := logger.GetReqInfo(ctx)
+
+	if reqInfo.API != api {
+		t.Errorf("Expected %s got %s", api, reqInfo.API)
+	}
+
+	if reqInfo.BucketName != args.BucketName {
+		t.Errorf("Expected %s got %s", args.BucketName, reqInfo.BucketName)
+	}
+}

cmd/web-handlers.go

@@ -74,31 +74,9 @@ type ServerInfoRep struct {
 	UIVersion       string `json:"uiVersion"`
 }
 
-// newWebContext creates a context with ReqInfo values from the given
-// http request and api name.
-func newWebContext(r *http.Request, api string) context.Context {
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	object := vars["object"]
-	prefix := vars["prefix"]
-
-	if prefix != "" {
-		object = prefix
-	}
-	reqInfo := &logger.ReqInfo{
-		DeploymentID: globalDeploymentID,
-		RemoteHost:   handlers.GetSourceIP(r),
-		UserAgent:    r.UserAgent(),
-		API:          api,
-		BucketName:   bucket,
-		ObjectName:   object,
-	}
-	return logger.SetReqInfo(context.Background(), reqInfo)
-}
-
 // ServerInfo - get server info.
 func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, reply *ServerInfoRep) error {
-	ctx := newWebContext(r, "webServerInfo")
+	ctx := newWebContext(r, args, "webServerInfo")
 	_, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
 		return toJSONError(ctx, authErr)
@@ -148,7 +126,7 @@ type StorageInfoRep struct {
 
 // StorageInfo - web call to gather storage usage statistics.
 func (web *webAPIHandlers) StorageInfo(r *http.Request, args *WebGenericArgs, reply *StorageInfoRep) error {
-	ctx := newWebContext(r, "webStorageInfo")
+	ctx := newWebContext(r, args, "webStorageInfo")
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		return toJSONError(ctx, errServerNotInitialized)
@@ -169,7 +147,7 @@ type MakeBucketArgs struct {
 
 // MakeBucket - creates a new bucket.
 func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, reply *WebGenericRep) error {
-	ctx := newWebContext(r, "webMakeBucket")
+	ctx := newWebContext(r, args, "webMakeBucket")
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		return toJSONError(ctx, errServerNotInitialized)
@@ -230,7 +208,7 @@ type RemoveBucketArgs struct {
 
 // DeleteBucket - removes a bucket, must be empty.
 func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs, reply *WebGenericRep) error {
-	ctx := newWebContext(r, "webDeleteBucket")
+	ctx := newWebContext(r, args, "webDeleteBucket")
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		return toJSONError(ctx, errServerNotInitialized)
@@ -318,7 +296,7 @@ type WebBucketInfo struct {
 
 // ListBuckets - list buckets api.
 func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, reply *ListBucketsRep) error {
-	ctx := newWebContext(r, "webListBuckets")
+	ctx := newWebContext(r, args, "webListBuckets")
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		return toJSONError(ctx, errServerNotInitialized)
@@ -421,7 +399,7 @@ type WebObjectInfo struct {
 
 // ListObjects - list objects api.
 func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, reply *ListObjectsRep) error {
-	ctx := newWebContext(r, "webListObjects")
+	ctx := newWebContext(r, args, "webListObjects")
 	reply.UIVersion = browser.UIVersion
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
@@ -616,7 +594,7 @@ type RemoveObjectArgs struct {
 
 // RemoveObject - removes an object, or all the objects at a given prefix.
 func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs, reply *WebGenericRep) error {
-	ctx := newWebContext(r, "webRemoveObject")
+	ctx := newWebContext(r, args, "webRemoveObject")
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		return toJSONError(ctx, errServerNotInitialized)
@@ -776,7 +754,7 @@ type LoginRep struct {
 
 // Login - user login handler.
 func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error {
-	ctx := newWebContext(r, "webLogin")
+	ctx := newWebContext(r, args, "webLogin")
 	token, err := authenticateWeb(args.Username, args.Password)
 	if err != nil {
 		return toJSONError(ctx, err)
@@ -795,7 +773,7 @@ type GenerateAuthReply struct {
 }
 
 func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, reply *GenerateAuthReply) error {
-	ctx := newWebContext(r, "webGenerateAuth")
+	ctx := newWebContext(r, args, "webGenerateAuth")
 	_, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
 		return toJSONError(ctx, authErr)
@@ -830,7 +808,7 @@ type SetAuthReply struct {
 
 // SetAuth - Set accessKey and secretKey credentials.
 func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *SetAuthReply) error {
-	ctx := newWebContext(r, "webSetAuth")
+	ctx := newWebContext(r, args, "webSetAuth")
 	claims, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
 		return toJSONError(ctx, authErr)
@@ -919,7 +897,7 @@ type URLTokenReply struct {
 
 // CreateURLToken creates a URL token (short-lived) for GET requests.
 func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs, reply *URLTokenReply) error {
-	ctx := newWebContext(r, "webCreateURLToken")
+	ctx := newWebContext(r, args, "webCreateURLToken")
 	claims, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
 		return toJSONError(ctx, authErr)
@@ -1503,7 +1481,7 @@ type GetBucketPolicyRep struct {
 
 // GetBucketPolicy - get bucket policy for the requested prefix.
 func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolicyArgs, reply *GetBucketPolicyRep) error {
-	ctx := newWebContext(r, "webGetBucketPolicy")
+	ctx := newWebContext(r, args, "webGetBucketPolicy")
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		return toJSONError(ctx, errServerNotInitialized)
@@ -1513,6 +1491,7 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic
 	if authErr != nil {
 		return toJSONError(ctx, authErr)
 	}
+
 	// For authenticated users apply IAM policy.
 	if !globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     claims.Subject,
@@ -1599,18 +1578,25 @@ type ListAllBucketPoliciesRep struct {
 
 // ListAllBucketPolicies - get all bucket policy.
 func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllBucketPoliciesArgs, reply *ListAllBucketPoliciesRep) error {
-	ctx := newWebContext(r, "WebListAllBucketPolicies")
+	ctx := newWebContext(r, args, "WebListAllBucketPolicies")
 	objectAPI := web.ObjectAPI()
 	if objectAPI == nil {
 		return toJSONError(ctx, errServerNotInitialized)
 	}
 
-	_, owner, authErr := webRequestAuthenticate(r)
+	claims, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
 		return toJSONError(ctx, authErr)
 	}
 
-	if !owner {
+	// For authenticated users apply IAM policy.
+	if !globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     claims.Subject,
+		Action:          iampolicy.GetBucketPolicyAction,
+		BucketName:      args.BucketName,
+		ConditionValues: getConditionValues(r, "", claims.Subject),
+		IsOwner:         owner,
+	}) {
 		return toJSONError(ctx, errAccessDenied)
 	}
 
@@ -1680,7 +1666,7 @@ type SetBucketPolicyWebArgs struct {
 
 // SetBucketPolicy - set bucket policy.
 func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolicyWebArgs, reply *WebGenericRep) error {
-	ctx := newWebContext(r, "webSetBucketPolicy")
+	ctx := newWebContext(r, args, "webSetBucketPolicy")
 	objectAPI := web.ObjectAPI()
 	reply.UIVersion = browser.UIVersion
 
@@ -1832,7 +1818,7 @@ type PresignedGetRep struct {
 
 // PresignedGET - returns presigned-Get url.
 func (web *webAPIHandlers) PresignedGet(r *http.Request, args *PresignedGetArgs, reply *PresignedGetRep) error {
-	ctx := newWebContext(r, "webPresignedGet")
+	ctx := newWebContext(r, args, "webPresignedGet")
 	claims, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
 		return toJSONError(ctx, authErr)

cmd/xl-sets.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
@@ -518,6 +519,21 @@ func (s *xlSets) DeleteBucketPolicy(ctx context.Context, bucket string) error {
 	return removePolicyConfig(ctx, s, bucket)
 }
 
+// SetBucketLifecycle sets lifecycle on bucket
+func (s *xlSets) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
+	return saveLifecycleConfig(ctx, s, bucket, lifecycle)
+}
+
+// GetBucketLifecycle will get lifecycle on bucket
+func (s *xlSets) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
+	return getLifecycleConfig(s, bucket)
+}
+
+// DeleteBucketLifecycle deletes all lifecycle on bucket
+func (s *xlSets) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
+	return removeLifecycleConfig(ctx, s, bucket)
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (s *xlSets) IsNotificationSupported() bool {
 	return s.getHashedSet("").IsNotificationSupported()

cmd/xl-v1-bucket.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -308,6 +309,21 @@ func (xl xlObjects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 	return removePolicyConfig(ctx, xl, bucket)
 }
 
+// SetBucketLifecycle sets lifecycle on bucket
+func (xl xlObjects) SetBucketLifecycle(ctx context.Context, bucket string, lifecycle *lifecycle.Lifecycle) error {
+	return saveLifecycleConfig(ctx, xl, bucket, lifecycle)
+}
+
+// GetBucketLifecycle will get lifecycle on bucket
+func (xl xlObjects) GetBucketLifecycle(ctx context.Context, bucket string) (*lifecycle.Lifecycle, error) {
+	return getLifecycleConfig(xl, bucket)
+}
+
+// DeleteBucketLifecycle deletes all lifecycle on bucket
+func (xl xlObjects) DeleteBucketLifecycle(ctx context.Context, bucket string) error {
+	return removeLifecycleConfig(ctx, xl, bucket)
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (xl xlObjects) IsNotificationSupported() bool {
 	return true

cmd/xl-v1-healing-common.go

@@ -183,10 +183,10 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			// it needs healing too.
 			for _, part := range partsMetadata[i].Parts {
 				checksumInfo := erasureInfo.GetChecksumInfo(part.Name)
-				err = onlineDisk.VerifyFile(bucket, pathJoin(object, part.Name), checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
+				err = onlineDisk.VerifyFile(bucket, pathJoin(object, part.Name), part.Size == 0, checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
 				if err != nil {
 					isCorrupt := strings.HasPrefix(err.Error(), "Bitrot verification mismatch - expected ")
-					if !isCorrupt && err != errFileNotFound && err != errVolumeNotFound {
+					if !isCorrupt && err != errFileNotFound && err != errVolumeNotFound && err != errFileUnexpectedSize {
 						logger.LogIf(ctx, err)
 					}
 					dataErrs[i] = err

cmd/xl-v1-healing.go

@@ -195,6 +195,9 @@ func shouldHealObjectOnDisk(xlErr, dataErr error, meta xlMetaV1, quorumModTime t
 		if dataErr == errFileNotFound {
 			return true
 		}
+		if dataErr == errFileUnexpectedSize {
+			return true
+		}
 		if _, ok := dataErr.(HashMismatchError); ok {
 			return true
 		}

cmd/xl-v1-healing_test.go

@@ -107,7 +107,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
-	// Remove the object backend files from the first disk.
+	// Test 1: Remove the object backend files from the first disk.
 	xl := obj.(*xlObjects)
 	firstDisk := xl.storageDisks[0]
 	err = firstDisk.DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
@@ -125,7 +125,34 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Errorf("Expected xl.json file to be present but stat failed - %v", err)
 	}
 
-	// Delete xl.json from more than read quorum number of disks, to create a corrupted situation.
+	// Test 2: Heal when part.1 is empty
+	partSt1, err := firstDisk.StatFile(bucket, filepath.Join(object, "part.1"))
+	if err != nil {
+		t.Errorf("Expected part.1 file to be present but stat failed - %v", err)
+	}
+	err = firstDisk.DeleteFile(bucket, filepath.Join(object, "part.1"))
+	if err != nil {
+		t.Errorf("Failure during part.1 removal - %v", err)
+	}
+	err = firstDisk.AppendFile(bucket, filepath.Join(object, "part.1"), []byte{})
+	if err != nil {
+		t.Errorf("Failure during creating part.1 - %v", err)
+	}
+	_, err = obj.HealObject(context.Background(), bucket, object, false, true, madmin.HealDeepScan)
+	if err != nil {
+		t.Errorf("Expected nil but received %v", err)
+	}
+	partSt2, err := firstDisk.StatFile(bucket, filepath.Join(object, "part.1"))
+	if err != nil {
+		t.Errorf("Expected from part.1 file to be present but stat failed - %v", err)
+	}
+	if partSt1.Size != partSt2.Size {
+		t.Errorf("part.1 file size is not the same before and after heal")
+	}
+
+	// Test 3: checks if HealObject returns an error when xl.json is not found
+	// in more than read quorum number of disks, to create a corrupted situation.
+
 	for i := 0; i <= len(xl.storageDisks)/2; i++ {
 		xl.storageDisks[i].DeleteFile(bucket, filepath.Join(object, xlMetaJSONFile))
 	}

dockerscripts/check-user.go

@@ -1,135 +0,0 @@
-// +build ignore
-
-/*
- * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package main
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"os/user"
-	"syscall"
-
-	"github.com/minio/cli"
-	minio "github.com/minio/minio/cmd"
-)
-
-var defaultUserGroup string
-
-func init() {
-	username := os.Getenv("MINIO_USERNAME")
-	groupname := os.Getenv("MINIO_GROUPNAME")
-	defaultUserGroup = username + ":" + groupname
-}
-
-func getUserGroup(path string) (string, error) {
-	fi, err := os.Stat(minio.PathJoin(path, ".minio.sys"))
-	if err != nil {
-		// Fresh directory we should default to what was requested by user.
-		if os.IsNotExist(err) {
-			fi, err = os.Stat(path)
-			if err != nil {
-				return "", err
-			}
-		} else {
-			return "", err
-		}
-	}
-	stat, ok := fi.Sys().(*syscall.Stat_t)
-	if !ok {
-		// Unable to figure out uid/gid, default to defaultUserGroup
-		return defaultUserGroup, nil
-	}
-	u, err := user.LookupId(fmt.Sprintf("%d", stat.Uid))
-	if err != nil {
-		return fmt.Sprintf("%d:%d", stat.Uid, stat.Gid), nil
-	}
-	g, err := user.LookupGroupId(fmt.Sprintf("%d", stat.Gid))
-	if err != nil {
-		return fmt.Sprintf("%d:%d", stat.Uid, stat.Gid), nil
-	}
-	return fmt.Sprintf("%s:%s", u.Username, g.Name), nil
-}
-
-func main() {
-	app := cli.NewApp()
-	app.Flags = append(minio.ServerFlags, minio.GlobalFlags...)
-	app.Action = func(ctx *cli.Context) {
-		// Fetch address option
-		serverAddr := ctx.GlobalString("address")
-		if serverAddr == "" || serverAddr == ":9000" {
-			serverAddr = ctx.String("address")
-		}
-		if ctx.Args().First() == "help" {
-			cli.ShowCommandHelpAndExit(ctx, "check-user", 1)
-		}
-		if ctx.Args().First() != "minio" {
-			cli.ShowCommandHelpAndExit(ctx, "check-user", 1)
-		}
-		args := cli.Args(ctx.Args().Tail())
-		if !args.Present() {
-			cli.ShowCommandHelpAndExit(ctx, "check-user", 1)
-		}
-		var ug string
-		var err error
-		switch args.First() {
-		case "gateway":
-			args = cli.Args(args.Tail())
-			if args.First() != "nas" {
-				fmt.Println(defaultUserGroup)
-				return
-			}
-			args = cli.Args(args.Tail())
-			if args.First() == "" {
-				fmt.Println("")
-				return
-			}
-			ug, err = getUserGroup(args.First())
-			if err != nil {
-				log.Fatalln(err)
-			}
-		case "server":
-			var setArgs [][]string
-			setArgs, err = minio.GetAllSets(args.Tail()...)
-			if err != nil {
-				log.Fatalln(err)
-			}
-			var endpoints minio.EndpointList
-			_, endpoints, _, err = minio.CreateEndpoints(serverAddr, setArgs...)
-			if err != nil {
-				log.Fatalln(err)
-			}
-			for _, endpoint := range endpoints {
-				if !endpoint.IsLocal {
-					continue
-				}
-				ug, err = getUserGroup(endpoint.Path)
-				if err != nil {
-					log.Fatalln(err)
-				}
-				break
-			}
-		default:
-			cli.ShowCommandHelpAndExit(ctx, "check-user", 1)
-		}
-		fmt.Println(ug)
-	}
-	if err := app.Run(os.Args); err != nil {
-		log.Fatalln(err)
-	}
-}

dockerscripts/docker-entrypoint.sh

@@ -15,9 +15,6 @@
 # limitations under the License.
 #
 
-export MINIO_USERNAME=${MINIO_USERNAME:-"minio"}
-export MINIO_GROUPNAME=${MINIO_GROUPNAME:-"minio"}
-
 # If command starts with an option, prepend minio.
 if [ "${1}" != "minio" ]; then
     if [ -n "${1}" ]; then
@@ -42,37 +39,35 @@ docker_secrets_env() {
     fi
 }
 
-## Create UID/GID based on available environment variables.
-docker_set_uid_gid() {
-    addgroup -S "$MINIO_GROUPNAME" >/dev/null 2>&1 && \
-        adduser -S -G "$MINIO_GROUPNAME" "$MINIO_USERNAME" >/dev/null 2>&1
+## Set SSE_MASTER_KEY from docker secrets if provided
+docker_sse_encryption_env() {
+    SSE_MASTER_KEY_FILE="/run/secrets/$MINIO_SSE_MASTER_KEY_FILE"
+
+    if [ -f "$SSE_MASTER_KEY_FILE" ]; then
+        MINIO_SSE_MASTER_KEY="$(cat "$SSE_MASTER_KEY_FILE")"
+        export MINIO_SSE_MASTER_KEY
+
+    fi
 }
 
-# su-exec to requested user, if user cannot be requested
-# existing user is used automatically.
+# su-exec to requested user, if service cannot run exec will fail.
 docker_switch_user() {
-    owner=$(check-user "$@")
-    if [ "${owner}" != "${MINIO_USERNAME}:${MINIO_GROUPNAME}" ]; then
-        ## Print the message only if we are not using non-default username:groupname.
-        if [ "${MINIO_USERNAME}:${MINIO_GROUPNAME}" != "minio:minio" ]; then
-            echo "Requested username/group ${MINIO_USERNAME}:${MINIO_GROUPNAME} cannot be used"
-            echo "Found existing data with user ${owner}, we will continue and use ${owner} instead."
-            return
-        fi
+    if [ ! -z "${MINIO_USERNAME}" ] && [ ! -z "${MINIO_GROUPNAME}" ]; then
+        addgroup -S "$MINIO_GROUPNAME" >/dev/null 2>&1 && \
+            adduser -S -G "$MINIO_GROUPNAME" "$MINIO_USERNAME" >/dev/null 2>&1
+
+        exec su-exec "${MINIO_USERNAME}:${MINIO_GROUPNAME}" "$@"
+    else
+        # fallback
+        exec "$@"
     fi
-    # check if su-exec is allowed, if yes proceed proceed.
-    if su-exec "${owner}" "/bin/ls" >/dev/null 2>&1; then
-        exec su-exec "${owner}" "$@"
-    fi
-    # fallback
-    exec "$@"
 }
 
 ## Set access env from secrets if necessary.
 docker_secrets_env
 
-## User Input UID and GID
-docker_set_uid_gid
+## Set sse encryption from secrets if necessary.
+docker_sse_encryption_env
 
 ## Switch to user if applicable.
 docker_switch_user "$@"

docs/bucket/notifications/README.md

@@ -286,12 +286,14 @@ MinIO requires a 5.x series version of Elasticsearch. This is the latest major r
 
 The MinIO server configuration file is stored on the backend in json format. The Elasticsearch configuration is located in the `elasticsearch` key under the `notify` top-level key. Create a configuration key-value pair here for your Elasticsearch instance. The key is a name for your Elasticsearch endpoint, and the value is a collection of key-value parameters described in the table below.
 
-| Parameter | Type     | Description                                                                                                                                                                                   |
-| :-------- | :------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `enable`  | _bool_   | (Required) Is this server endpoint configuration active/enabled?                                                                                                                              |
-| `format`  | _string_ | (Required) Either `namespace` or `access`.                                                                                                                                                    |
-| `url`     | _string_ | (Required) The Elasticsearch server's address, with optional authentication info. For example: `http://localhost:9200` or with authentication info `http://elastic:MagicWord@127.0.0.1:9200`. |
-| `index`   | _string_ | (Required) The name of an Elasticsearch index in which MinIO will store documents.                                                                                                            |
+| Parameter    | Type     | Description                                                                                                                                                                                   |
+| :----------- | :------- | :-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `enable`     | _bool_   | (Required) Is this server endpoint configuration active/enabled?                                                                                                                              |
+| `format`     | _string_ | (Required) Either `namespace` or `access`.                                                                                                                                                    |
+| `url`        | _string_ | (Required) The Elasticsearch server's address, with optional authentication info. For example: `http://localhost:9200` or with authentication info `http://elastic:MagicWord@127.0.0.1:9200`. |
+| `index`      | _string_ | (Required) The name of an Elasticsearch index in which MinIO will store documents.                                                                                                            |
+| `queueDir`   | _string_ | Persistent store for events when Elasticsearch broker is offline                                                                                                                              |
+| `queueLimit` | _int_    | Set the maximum event limit for the persistent store. The default limit is 10000                                                                                                              |
 
 An example of Elasticsearch configuration is as follows:
 
@@ -301,11 +303,15 @@ An example of Elasticsearch configuration is as follows:
         "enable": true,
         "format": "namespace",
         "url": "http://127.0.0.1:9200",
-        "index": "minio_events"
+        "index": "minio_events",
+        "queueDir": "",
+        "queueLimit": 0
     }
 },
 ```
 
+Minio supports persistent event store. The persistent store will backup events when the Elasticsearch broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
+
 If Elasticsearch has authentication enabled, the credentials can be supplied to MinIO via the `url` parameter formatted as `PROTO://USERNAME:PASSWORD@ELASTICSEARCH_HOST:PORT`.
 
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
@@ -454,11 +460,15 @@ An example of Redis configuration is as follows:
         "format": "namespace",
         "address": "127.0.0.1:6379",
         "password": "yoursecret",
-        "key": "bucketevents"
+        "key": "bucketevents",
+        "queueDir": "",
+        "queueLimit": 0
     }
 }
 ```
 
+MinIO supports persistent event store. The persistent store will backup events when the Redis broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
+
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
 
 ```sh
@@ -537,11 +547,12 @@ The NATS configuration block in `config.json` is as follows:
         "password": "yoursecret",
         "token": "",
         "secure": false,
-        "pingInterval": 0
+        "pingInterval": 0,
+        "queueDir": "",
+        "queueLimit": 0,
         "streaming": {
             "enable": false,
             "clusterID": "",
-            "clientID": "",
             "async": false,
             "maxPubAcksInflight": 0
         }
@@ -549,6 +560,8 @@ The NATS configuration block in `config.json` is as follows:
 },
 ```
 
+MinIO supports persistent event store. The persistent store will backup events when the NATS broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
+
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
 
 ```sh
@@ -561,7 +574,7 @@ After updating the NATS configuration in /tmp/myconfig , use `mc admin config se
 $ mc admin config set myminio < /tmp/myconfig
 ```
 
-MinIO server also supports [NATS Streaming mode](http://nats.io/documentation/streaming/nats-streaming-intro/) that offers additional functionality like `Message/event persistence`, `At-least-once-delivery`, and `Publisher rate limiting`. To configure MinIO server to send notifications to NATS Streaming server, update the MinIO server configuration file as follows:
+MinIO server also supports [NATS Streaming mode](http://nats.io/documentation/streaming/nats-streaming-intro/) that offers additional functionality like `At-least-once-delivery`, and `Publisher rate limiting`. To configure MinIO server to send notifications to NATS Streaming server, update the MinIO server configuration file as follows:
 
 ```
 "nats": {
@@ -574,10 +587,11 @@ MinIO server also supports [NATS Streaming mode](http://nats.io/documentation/st
         "token": "",
         "secure": false,
         "pingInterval": 0,
+        "queueDir": "",
+        "queueLimit": 0,
         "streaming": {
             "enable": true,
             "clusterID": "test-cluster",
-            "clientID": "minio-client",
             "async": true,
             "maxPubAcksInflight": 10
         }
@@ -667,20 +681,47 @@ import (
 )
 
 func main() {
-	natsConnection, _ := stan.Connect("test-cluster", "test-client")
-	log.Println("Connected")
+
+	var stanConnection stan.Conn
+
+	subscribe := func() {
+		fmt.Printf("Subscribing to subject 'bucketevents'\n")
+		stanConnection.Subscribe("bucketevents", func(m *stan.Msg) {
+
+			// Handle the message
+			fmt.Printf("Received a message: %s\n", string(m.Data))
+		})
+	}
+
+
+	stanConnection, _ = stan.Connect("test-cluster", "test-client", stan.NatsURL("nats://yourusername:yoursecret@0.0.0.0:4222"), stan.SetConnectionLostHandler(func(c stan.Conn, _ error) {
+		go func() {
+			for {
+				// Reconnect if the connection is lost.
+				if stanConnection == nil || stanConnection.NatsConn() == nil ||  !stanConnection.NatsConn().IsConnected() {
+					stanConnection, _ = stan.Connect("test-cluster", "test-client", stan.NatsURL("nats://yourusername:yoursecret@0.0.0.0:4222"), stan.SetConnectionLostHandler(func(c stan.Conn, _ error) {
+						if c.NatsConn() != nil {
+							c.NatsConn().Close()
+						}
+						_ = c.Close()
+					}))
+					if stanConnection != nil {
+						subscribe()
+					}
+
+				}
+			}
+
+		}()
+	}))
 
 	// Subscribe to subject
-	log.Printf("Subscribing to subject 'bucketevents'\n")
-	natsConnection.Subscribe("bucketevents", func(m *stan.Msg) {
-
-		// Handle the message
-		fmt.Printf("Received a message: %s\n", string(m.Data))
-	})
+	subscribe()
 
 	// Keep the connection alive
 	runtime.Goexit()
 }
+
 ```
 
 ```
@@ -947,6 +988,7 @@ The MinIO server configuration file is stored on the backend in json format. Upd
     }
 }
 ```
+
 MinIO supports persistent event store. The persistent store will backup events when the kafka broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
 
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
@@ -1104,12 +1146,17 @@ An example configuration for NSQ is shown below:
         "tls": {
             "enable": false,
             "skipVerify": true
-        }
+        },
+        "queueDir": "",
+        "queueLimit": 0
     }
 }
+
+MinIO supports persistent event store. The persistent store will backup events when the NSQ broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field and the maximum limit of events in the queueDir in `queueLimit` field. For eg, the `queueDir` can be `/home/events` and `queueLimit` can be `1000`. By default, the `queueLimit` is set to 10000.
+
 ```
 
-To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
+To update the configuration, use `mc admin config get` command to get the current configuration file for the MinIO deployment in json format, and save it locally.
 
 ```sh
 $ mc admin config get myminio/ > /tmp/myconfig

docs/config/config.sample.json

@@ -56,7 +56,9 @@
 				"enable": false,
 				"format": "",
 				"url": "",
-				"index": ""
+				"index": "",
+                                "queueDir": "",
+                                "queueLimit": 0
 			}
 		},
 		"kafka": {
@@ -115,6 +117,8 @@
 				"token": "",
 				"secure": false,
 				"pingInterval": 0,
+                                "queueDir": "",
+                                "queueLimit": 0,
 				"streaming": {
 					"enable": false,
 					"clusterID": "",
@@ -131,7 +135,9 @@
 				"tls": {
 					"enable": false,
 					"skipVerify": true
-				}
+				},
+                                "queueDir": "",
+                                "queueLimit": 0                 
 			}
 		},
 		"postgresql": {
@@ -153,7 +159,9 @@
 				"format": "",
 				"address": "",
 				"password": "",
-				"key": ""
+				"key": "",
+                                "queueDir": "",
+                                "queueLimit": 0                 
 			}
 		},
 		"webhook": {

docs/distributed/DESIGN.md

@@ -1,28 +1,26 @@
-# Large Bucket Support Design Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
-
-This document explains the design approach, advanced use cases and limits of the large bucket feature. If you're looking to get started with large bucket support, we suggest you go through the [getting started document](https://github.com/minio/minio/blob/master/docs/large-bucket/README.md) first.
+# Distributed Server Design Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io)
+This document explains the design approach, advanced use cases and limits of the MinIO distributed server.
 
 ## Command-line
-
 ```
 NAME:
-  minio server - Start object storage server.
+  minio server - start object storage server.
 
 USAGE:
   minio server [FLAGS] DIR1 [DIR2..]
   minio server [FLAGS] DIR{1...64}
 
 DIR:
-  DIR points to a directory on a filesystem. When you want to combine multiple drives
-  into a single large system, pass one directory per filesystem separated by space.
-  You may also use a `...` convention to abbreviate the directory arguments. Remote
-  directories in a distributed setup are encoded as HTTP(s) URIs.
+  DIR points to a directory on a filesystem. When you want to combine
+  multiple drives into a single large system, pass one directory per
+  filesystem separated by space. You may also use a '...' convention
+  to abbreviate the directory arguments. Remote directories in a
+  distributed setup are encoded as HTTP(s) URIs.
 ```
 
 ## Common usage
 
 Standalone erasure coded configuration with 4 sets with 16 disks each.
-
 ```
 minio server dir{1...64}
 ```
@@ -33,38 +31,21 @@ Distributed erasure coded configuration with 64 sets with 16 disks each.
 minio server http://host{1...16}/export{1...64}
 ```
 
-## Other usages
+## Architecture
 
-### Advanced use cases with multiple ellipses
+Expansion of ellipses and choice of erasure sets based on this expansion is an automated process in MinIO. Here are some of the details of our underlying erasure coding behavior.
 
-Standalone erasure coded configuration with 4 sets with 16 disks each, which spawns disks across controllers.
-```
-minio server /mnt/controller{1...4}/data{1...16}
-```
+- Erasure coding used by MinIO is [Reed-Solomon](https://github.com/klauspost/reedsolomon) erasure coding scheme, which has a total shard maximum of 256 i.e 128 data and 128 parity. MinIO design goes beyond this limitation by doing some practical architecture choices.
 
-Standalone erasure coded configuration with 16 sets 16 disks per set, across mnts, across controllers.
-```
-minio server /mnt{1..4}/controller{1...4}/data{1...16}
-```
+- Erasure set is a single erasure coding unit within a MinIO deployment. An object is sharded within an erasure set. Erasure set size is automatically calculated based on the number of disks. MinIO supports unlimited number of disks but each erasure set can be upto 16 disks and a minimum of 4 disks.
 
-Distributed erasure coded configuration with 2 sets 16 disks per set across hosts.
-```
-minio server http://host{1...32}/disk1
-```
+- We limited the number of drives to 16 for erasure set because, erasure code shards more than 16 can become chatty and do not have any performance advantages. Additionally since 16 drive erasure set gives you tolerance of 8 disks per object by default which is plenty in any practical scenario.
 
-Distributed erasure coded configuration with rack level redundancy 32 sets in total, 16 disks per set.
-```
-minio server http://rack{1...4}-host{1...8}.example.net/export{1...16}
-```
+- Choice of erasure set size is automatic based on the number of disks available, let's say for example if there are 32 servers and 32 disks which is a total of 1024 disks. In this scenario 16 becomes the erasure set size. This is decided based on the greatest common divisor (GCD) of acceptable erasure set sizes ranging from *4, 6, 8, 10, 12, 14, 16*.
 
-Distributed erasure coded configuration with no rack level redundancy but redundancy with in the rack we split the arguments, 32 sets in total, 16 disks per set.
-```
-minio server http://rack1-host{1...8}.example.net/export{1...16} http://rack2-host{1...8}.example.net/export{1...16} http://rack3-host{1...8}.example.net/export{1...16} http://rack4-host{1...8}.example.net/export{1...16}
-```
+- *If total disks has many common divisors the algorithm chooses the minimum amounts of erasure sets possible for a erasure set size of any N*.  In the example with 1024 disks - 4, 8, 16 are GCD factors. With 16 disks we get a total of 64 possible sets, with 8 disks we get a total of 128 possible sets, with 4 disks we get a total of 256 possible sets. So algorithm automatically chooses 64 sets, which is *16 * 64 = 1024* disks in total.
 
-### Expected expansion for double ellipses
-
-MinIO server internally expands ellipses passed as arguments. Here is a sample expansion to demonstrate the process
+- In this algorithm, we also make sure that we spread the disks out evenly. MinIO server expands ellipses passed as arguments. Here is a sample expansion to demonstrate the process.
 
 ```
 minio server http://host{1...4}/export{1...8}
@@ -106,6 +87,50 @@ Expected expansion
 > http://host4/export8
 ```
 
+A noticeable trait of this expansion is that it chooses unique hosts such that the erasure code is efficient across drives and hosts.
+
+- Choosing an erasure set for the object is decided during `PutObject()`, object names are used to find the right erasure set using the following pseudo code.
+```go
+// hashes the key returning an integer.
+func crcHashMod(key string, cardinality int) int {
+        keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable)
+        return int(keyCrc % uint32(cardinality))
+}
+```
+Input for the key is the object name specified in `PutObject()`, returns a unique index. This index is one of the erasure sets where the object will reside. This function is a consistent hash for a given object name i.e for a given object name the index returned is always the same.
+
+- Write and Read quorum are required to be satisfied only across the erasure set for an object. Healing is also done per object within the erasure set which contains the object.
+
+- MinIO does erasure coding at the object level not at the volume level, unlike other object storage vendors. This allows applications to choose different storage class by setting `x-amz-storage-class=STANDARD/REDUCED_REDUNDANCY` for each object uploads so effectively utilizing the capacity of the cluster. Additionally these can also be enforced using IAM policies to make sure the client uploads with correct HTTP headers.
+
+## Other usages
+
+### Advanced use cases with multiple ellipses
+
+Standalone erasure coded configuration with 4 sets with 16 disks each, which spawns disks across controllers.
+```
+minio server /mnt/controller{1...4}/data{1...16}
+```
+
+Standalone erasure coded configuration with 16 sets, 16 disks per set, across mounts and controllers.
+```
+minio server /mnt{1..4}/controller{1...4}/data{1...16}
+```
+
+Distributed erasure coded configuration with 2 sets, 16 disks per set across hosts.
+```
+minio server http://host{1...32}/disk1
+```
+
+Distributed erasure coded configuration with rack level redundancy 32 sets in total, 16 disks per set.
+```
+minio server http://rack{1...4}-host{1...8}.example.net/export{1...16}
+```
+
+Distributed erasure coded configuration with no rack level redundancy but redundancy with in the rack we split the arguments, 32 sets in total, 16 disks per set.
+```
+minio server http://rack1-host{1...8}.example.net/export{1...16} http://rack2-host{1...8}.example.net/export{1...16} http://rack3-host{1...8}.example.net/export{1...16} http://rack4-host{1...8}.example.net/export{1...16}
+```
 ## Backend `format.json` changes
 
 `format.json` has new fields
@@ -186,5 +211,5 @@ type formatXLV2 struct {
 
 ## Limits
 
-- Minimum of 4 disks are needed for erasure coded configuration.
+- Minimum of 4 disks are needed for any erasure coded configuration.
 - Maximum of 32 distinct nodes are supported in distributed configuration.

docs/distributed/README.md

@@ -67,7 +67,6 @@ __NOTE:__ `{1...n}` shown have 3 dots! Using only 2 dots `{1..32}` will be inter
 To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/docs/minio-client-quickstart-guide).
 
 ## Explore Further
-- [MinIO Large Bucket Support Guide](https://docs.min.io/docs/minio-large-bucket-support-quickstart-guide)
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
 - [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)

docs/docker/README.md

@@ -58,20 +58,19 @@ docker run -p 9000:9000 --name minio1 \
   minio/minio server /data
 ```
 
-### Run MinIO Docker as non root user
-MinIO server runs as non-root within the container by default. However, this is applicable only if you're deploying new MinIO instance (not upgrading from older releases). Deployments upgrading from older MinIO deployments, will continue to run as the user previously used if any.
+### Run MinIO Docker as regular user
+MinIO server doesn't run as a regular user by default in docker containers. To run MinIO container as regular user use environment variables `MINIO_USERNAME` and `MINIO_GROUPNAME`.
 
-By default `minio` is username and groupname. Use environment variables `MINIO_USERNAME` and `MINIO_GROUPNAME` to override these default values.
+> NOTE: If you are upgrading from existing deployments, you need to make sure this user has write access to previous persistent volumes. MinIO will not migrate the content automatically.
 
 #### GNU/Linux and macOS
 ```sh
 docker run -p 9000:9000 --name minio1 \
   -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  -e "MINIO_USERNAME=custom" \
-  -e "MINIO_GROUPNAME=custom" \
+  -e "MINIO_USERNAME=minio-user" \
+  -e "MINIO_GROUPNAME=minio-user" \
   -v /mnt/data:/data \
-  -v /mnt/config:/root/.minio \
   minio/minio server /data
 ```
 
@@ -80,10 +79,9 @@ docker run -p 9000:9000 --name minio1 \
 docker run -p 9000:9000 --name minio1 \
   -e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  -e "MINIO_USERNAME=custom" \
-  -e "MINIO_GROUPNAME=custom" \
+  -e "MINIO_USERNAME=minio-user" \
+  -e "MINIO_GROUPNAME=minio-user" \
   -v D:\data:/data \
-  -v D:\minio\config:/root/.minio \
   minio/minio server /data
 ```
 

docs/kms/README.md

@@ -12,17 +12,17 @@ MinIO supports two different KMS concepts:
    by enabling or disabling the corresponding master keys on demand.
 
 - Direct KMS master keys:
-   MinIO can also be configured to directly use a master key specified by the environment variable `MINIO_SSE_MASTER_KEY`.
+   MinIO can also be configured to directly use a master key specified by the environment variable `MINIO_SSE_MASTER_KEY` or with a docker secret key.
    Direct master keys are useful if the storage backend is not on the same machine as the MinIO server, e.g.,
-   if network drives or MinIO gateway is used and an external KMS would cause too much management overhead.  
-   
+   if network drives or MinIO gateway is used and an external KMS would cause too much management overhead.
+
    Note: KMS master keys are mainly for testing purposes. It's not recommended to use them for production deployments.
    Further if the MinIO server machine is ever compromised, then the master key must also be treated as compromised.
 
-**Important:**  
+**Important:**
 If multiple MinIO servers are configured as [gateways](https://github.com/minio/minio/blob/master/docs/gateway/README.md)
 pointing to the *same* backend - for example the same NAS storage - then the KMS configuration **must** be the same for
-all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It is the operators' 
+all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It is the operators'
 responsibility to ensure consistency.
 
 ## Get started
@@ -36,47 +36,135 @@ Either use Hashicorp Vault as external KMS or specify a master key directly depe
 
 #### 2.1 Setup Hashicorp Vault
 
-Here is a sample quick start for configuring vault with a transit backend and Approle with correct policy 
+Here is a sample quick start for configuring vault with a transit backend and Approle with correct policy
 
 MinIO requires the following Vault setup:
 - The [transit backend](https://www.vaultproject.io/api/secret/transit/index.html) configured with a named encryption key-ring
 - [AppRole](https://www.vaultproject.io/docs/auth/approle.html) based authentication with read/update policy for transit backend. In particular, read and update policy are required for the [Generate Data Key](https://www.vaultproject.io/api/secret/transit/index.html#generate-data-key) endpoint and [Decrypt Data](https://www.vaultproject.io/api/secret/transit/index.html#decrypt-data) endpoint.
 
-**2.1.1 Start Vault server in dev mode**
-
-In dev mode, Vault server runs in-memory and starts unsealed. Note that running Vault in dev mode is insecure and any data stored in the Vault is lost upon restart.
+**2.1.1 Start Vault server**
 
+Vault requires access to `mlock` syscall by default. Use `setcap` to give the Vault executable the ability to use the `mlock` syscall without running the process as root.  To do so run:
 ```
-vault server -dev
+sudo setcap cap_ipc_lock=+ep $(readlink -f $(which vault))
 ```
 
-**2.1.2 Set up vault transit backend and create an app role**
+Create `vault-config.json` to use file backend and start the server.
+```
+cat > vault-config.json <<EOF
+{
+  "api_addr": "http://127.0.0.1:8200",
+  "backend": {
+    "file": {
+      "path": "vault/file"
+    }
+  },
+  "default_lease_ttl": "168h",
+  "max_lease_ttl": "720h",
+  "listener": {
+    "tcp": {
+      "address": "0.0.0.0:8200",
+      "tls_disable": true
+    }
+  },
+  "ui": true
+}
+EOF
+
+vault server -config vault-config.json
+```
+
+> NOTE: In this example we use `"tls_disable": true` for demonstration purposes only,
+> in production setup you should generate proper TLS certificates by specifying
+> - [`tls_cert_file`](https://www.vaultproject.io/docs/configuration/listener/tcp.html#tls_cert_file)
+> - [`tls_key_file`](https://www.vaultproject.io/docs/configuration/listener/tcp.html#tls_key_file)
+
+
+**2.1.2 Initialize vault and unseal it**
 
 ```
+export VAULT_ADDR='http://127.0.0.1:8200'
+vault operator init
+Unseal Key 1: eyW/+8ZtsgT81Cb0e8OVxzJAQP5lY7Dcamnze+JnWEDT
+Unseal Key 2: 0tZn+7QQCxphpHwTm6/dC3LpP5JGIbYl6PK8Sy79R+P2
+Unseal Key 3: cmhs+AUMXUuB6Lzsvgcbp3bRT6VDGQjgCBwB2xm0ANeF
+Unseal Key 4: /fTPpec5fWpGqWHK+uhnnTNMQyAbl5alUi4iq2yNgyqj
+Unseal Key 5: UPdDVPto+H6ko+20NKmagK40MOskqOBw4y/S51WpgVy/
+
+Initial Root Token: s.zaU4Gbcu0Wh46uj2V3VuUde0
+
+Vault is initialized with 5 key shares and a key threshold of 3. Please securely
+distribute the key shares printed above. When the Vault is re-sealed,
+restarted, or stopped, you must supply at least 3 of these keys to unseal it
+before it can start servicing requests.
+
+Vault does not store the generated master key. Without at least 3 key to
+reconstruct the master key, Vault will remain permanently sealed!
+
+It is possible to generate new unseal keys, provided you have a quorum of
+existing unseal keys shares. See "vault operator rekey" for more information.
+```
+
+Use any of the previously generated keys to unseal the vault
+```
+vault operator unseal eyW/+8ZtsgT81Cb0e8OVxzJAQP5lY7Dcamnze+JnWEDT
+vault operator unseal 0tZn+7QQCxphpHwTm6/dC3LpP5JGIbYl6PK8Sy79R+P2
+vault operator unseal cmhs+AUMXUuB6Lzsvgcbp3bRT6VDGQjgCBwB2xm0ANeF
+Key             Value
+---             -----
+Seal Type       shamir
+Initialized     true
+Sealed          false  ---> NOTE: vault is unsealed
+Total Shares    5
+Threshold       3
+Version         1.1.3
+Cluster Name    vault-cluster-3f084948
+Cluster ID      8c92e999-7062-4da6-4434-0fc05f34824d
+HA Enabled      false
+```
+
+Obtain root token from the `vault operator init` output above. It is displayed as `Initial Root Token: s.zaU4Gbcu0Wh46uj2V3VuUde0`
+
+**2.1.3 Set up vault transit backend and create an app role**
+```
+export VAULT_TOKEN=s.zaU4Gbcu0Wh46uj2V3VuUde0
+
+vault auth enable approle    # enable approle style auth
+vault secrets enable transit  # enable transit secrets engine
+# define an encryption key-ring for the transit path
+vault write -f  transit/keys/my-minio-key
+
 cat > vaultpolicy.hcl <<EOF
-path "transit/datakey/plaintext/my-minio-key" { 
+path "transit/datakey/plaintext/my-minio-key" {
   capabilities = [ "read", "update"]
 }
-path "transit/decrypt/my-minio-key" { 
+path "transit/decrypt/my-minio-key" {
   capabilities = [ "read", "update"]
 }
-path "transit/encrypt/my-minio-key" { 
+path "transit/encrypt/my-minio-key" {
   capabilities = [ "read", "update"]
 }
 
 EOF
 
-export VAULT_ADDR='http://127.0.0.1:8200'
-vault auth enable approle    # enable approle style auth
-vault secrets enable transit  # enable transit secrets engine
-vault write -f  transit/keys/my-minio-key  #define a encryption key-ring for the transit path
-vault policy write minio-policy ./vaultpolicy.hcl  #define a policy for AppRole to access transit path
-vault write auth/approle/role/my-role token_num_uses=0  secret_id_num_uses=0  period=5m # period indicates it is renewable if token is renewed before the period is over
+# define a policy for AppRole to access transit path
+vault policy write minio-policy ./vaultpolicy.hcl
+
+# period indicates it is renewable if token is renewed before the period is over
+vault write auth/approle/role/my-role token_num_uses=0  secret_id_num_uses=0  period=5m
+
 # define an AppRole
 vault write auth/approle/role/my-role policies=minio-policy # apply policy to role
 vault read auth/approle/role/my-role/role-id  # get Approle ID
-vault write -f auth/approle/role/my-role/secret-id
+Key        Value
+---        -----
+role_id    8c03926c-6c51-7a1d-cf7d-62e48ab8d6d7
 
+vault write -f auth/approle/role/my-role/secret-id
+Key                   Value
+---                   -----
+secret_id             edd8738c-6efe-c226-74f9-ef5b66e119d7
+secret_id_accessor    57d1db64-6350-c321-4a3e-fc6aeb7d00b6
 ```
 
 The AppRole ID, AppRole Secret Id, Vault endpoint and Vault key name can now be used to start minio server with Vault as KMS.
@@ -85,10 +173,10 @@ The AppRole ID, AppRole Secret Id, Vault endpoint and Vault key name can now be
 
 You'll need the Vault endpoint, AppRole ID, AppRole SecretID and encryption key-ring name defined in step 2.1.2
 
-```sh
-export MINIO_SSE_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
-export MINIO_SSE_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
-export MINIO_SSE_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
+```
+export MINIO_SSE_VAULT_APPROLE_ID=8c03926c-6c51-7a1d-cf7d-62e48ab8d6d7
+export MINIO_SSE_VAULT_APPROLE_SECRET=edd8738c-6efe-c226-74f9-ef5b66e119d7
+export MINIO_SSE_VAULT_ENDPOINT=http://vault-endpoint-ip:8200
 export MINIO_SSE_VAULT_KEY_NAME=my-minio-key
 export MINIO_SSE_VAULT_AUTH_TYPE=approle
 minio server ~/export
@@ -127,19 +215,38 @@ minio gateway s3
 
 #### 2.2 Specify a master key
 
+**2.2.1 KMS master key from environment variables**
+
 A KMS master key consists of a master-key ID (CMK) and the 256 bit master key encoded as HEX value separated by a `:`.
 A KMS master key can be specified directly using:
 
-```sh
+```
 export MINIO_SSE_MASTER_KEY=my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574
 ```
 
 Please use your own master key. A random master key can be generated using e.g. this command on Linux/Mac/BSD* systems:
 
-```sh
+```
 head -c 32 /dev/urandom | xxd -c 32 -ps
 ```
 
+**2.2.2 KMS master key from docker secret**
+
+Alternatively, you may pass a master key as a [Docker secret](https://docs.docker.com/engine/swarm/secrets/).
+
+```bash
+echo "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574" | docker secret create sse_master_key
+```
+
+Obviously, do not use this demo key for anything real!
+
+To use another secret name, follow the instructions above and replace sse_master_key with your custom names (e.g. my_sse_master_key).
+Then, set the MINIO_SSE_MASTER_KEY_FILE environment variable to your secret name:
+
+```bash
+export MINIO_SSE_MASTER_KEY_FILE=my_sse_master_key
+```
+
 ### 3. Test your setup
 To test this setup, start minio server with environment variables set in Step 3, and server is ready to handle SSE-S3 requests.
 
@@ -149,18 +256,18 @@ MinIO can also enable auto-encryption **if** a valid KMS configuration is specif
 encrypted objects. Auto-Encryption, if enabled, ensures that all uploaded objects are encrypted using the specified
 KMS configuration.
 
-Auto-Encryption is useful especially if the MinIO operator wants to ensure that objects are **never** stored in 
+Auto-Encryption is useful especially if the MinIO operator wants to ensure that objects are **never** stored in
 plaintext - for example if sensitive data is stored on public cloud storage.
 
 To enable auto-encryption set the environment variable to `on`:
 
-```sh
+```
 export MINIO_SSE_AUTO_ENCRYPTION=on
 ```
 
 To verify auto-encryption, use the `mc` command:
 
-```sh
+```
 mc cp test.file myminio/crypt/
 test.file:              5 B / 5 B  ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓  100.00% 337 B/s 0s
 mc stat myminio/crypt/test.file
@@ -171,8 +278,7 @@ Encrypted :
 ```
 
 Note: Auto-Encryption only affects non-SSE-C requests since objects uploaded using SSE-C are already encrypted
-and S3 only allows either SSE-S3 or SSE-C but not both for the same object.   
-
+and S3 only allows either SSE-S3 or SSE-C but not both for the same object.
 
 # Explore Further
 

docs/kms/vault-config.json

@@ -0,0 +1,17 @@
+{
+  "api_addr": "http://127.0.0.1:8200",
+  "backend": {
+    "file": {
+      "path": "vault/file"
+    }
+  },
+  "default_lease_ttl": "168h",
+  "max_lease_ttl": "720h",
+  "listener": {
+    "tcp": {
+      "address": "0.0.0.0:8200",
+      "tls_disable": true
+    }
+  },
+  "ui": true
+}

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+  image: minio/minio:RELEASE.2019-07-17T22-54-12Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+  image: minio/minio:RELEASE.2019-07-17T22-54-12Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+  image: minio/minio:RELEASE.2019-07-17T22-54-12Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+  image: minio/minio:RELEASE.2019-07-17T22-54-12Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+    image: minio/minio:RELEASE.2019-07-17T22-54-12Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+        image: minio/minio:RELEASE.2019-07-17T22-54-12Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+        image: minio/minio:RELEASE.2019-07-17T22-54-12Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+        image: minio/minio:RELEASE.2019-07-17T22-54-12Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-07-05T21-20-21Z
+        image: minio/minio:RELEASE.2019-07-17T22-54-12Z
         args:
         - server
         - /data

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/DataDog/zstd v1.4.0 // indirect
 	github.com/alecthomas/participle v0.2.1
 	github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5
+	github.com/aws/aws-sdk-go v1.20.21
 	github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f // indirect
 	github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2
 	github.com/cheggaaa/pb v1.0.28
@@ -27,6 +28,7 @@ require (
 	github.com/fatih/color v1.7.0
 	github.com/fatih/structs v1.1.0
 	github.com/go-sql-driver/mysql v1.4.1
+	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
 	github.com/golang/snappy v0.0.1
 	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/gorilla/handlers v1.4.0
@@ -52,12 +54,13 @@ require (
 	github.com/marstr/guid v1.1.0 // indirect
 	github.com/mattn/go-isatty v0.0.7
 	github.com/miekg/dns v1.1.8
-	github.com/minio/cli v1.20.0
+	github.com/minio/cli v1.21.0
 	github.com/minio/dsync/v2 v2.0.0
 	github.com/minio/hdfs/v3 v3.0.0
 	github.com/minio/highwayhash v1.0.0
 	github.com/minio/lsync v1.0.1
 	github.com/minio/mc v0.0.0-20190529152718-f4bb0b8850cb
+	github.com/minio/minio-go v0.0.0-20190327203652-5325257a208f
 	github.com/minio/minio-go/v6 v6.0.29
 	github.com/minio/parquet-go v0.0.0-20190318185229-9d767baf1679
 	github.com/minio/sha256-simd v0.1.0
@@ -73,6 +76,7 @@ require (
 	github.com/nats-io/stan.go v0.4.5
 	github.com/ncw/directio v1.0.5
 	github.com/nsqio/go-nsq v1.0.7
+	github.com/pkg/errors v0.8.1
 	github.com/pkg/profile v1.3.0
 	github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829
 	github.com/rcrowley/go-metrics v0.0.0-20190704165056-9c2d0518ed81 // indirect
@@ -80,7 +84,7 @@ require (
 	github.com/rs/cors v1.6.0
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/satori/go.uuid v1.2.0 // indirect
-	github.com/segmentio/go-prompt v1.2.1-0.20161017233205-f0d19b6901ad
+	github.com/sirupsen/logrus v1.4.2
 	github.com/skyrings/skyring-common v0.0.0-20160929130248-d1c0bb1cbd5e
 	github.com/streadway/amqp v0.0.0-20190402114354-16ed540749f6
 	github.com/tidwall/gjson v1.2.1
@@ -90,11 +94,9 @@ require (
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
 	go.etcd.io/bbolt v1.3.3 // indirect
 	go.uber.org/atomic v1.3.2
-	golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
-	golang.org/x/lint v0.0.0-20190409202823-959b441ac422 // indirect
-	golang.org/x/net v0.0.0-20190628185345-da137c7871d7 // indirect
+	golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4
+	golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect
 	golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb
-	golang.org/x/tools v0.0.0-20190703212419-2214986f1668 // indirect
 	google.golang.org/api v0.4.0
 	gopkg.in/Shopify/sarama.v1 v1.20.0
 	gopkg.in/olivere/elastic.v5 v5.0.80

go.sum

@@ -56,6 +56,8 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.31/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
+github.com/aws/aws-sdk-go v1.20.21 h1:22vHWL9rur+SRTYPHAXlxJMFIA9OSYsYDIAHFDhQ7Z0=
+github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
 github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
 github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM=
@@ -180,6 +182,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -336,6 +340,8 @@ github.com/jeffchao/backoff v0.0.0-20140404060208-9d7fd7aa17f2/go.mod h1:xkfESuH
 github.com/jefferai/jsonx v1.0.0/go.mod h1:OGmqmi2tTeI/PS+qQfBDToLHHJIy/RMp24fPo8vFvoQ=
 github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM=
+github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
@@ -376,8 +382,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/kurin/blazer v0.5.3 h1:SAgYv0TKU0kN/ETfO5ExjNAPyMt2FocO2s/UlCHfjAk=
-github.com/kurin/blazer v0.5.3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
 github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3 h1:1sl2HmNtqGnDuydLgCJwZIpDLGqZOdwOkcY8WtUl8Cw=
 github.com/kurin/blazer v0.5.4-0.20190613185654-cf2f27cc0be3/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
 github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
@@ -412,6 +416,8 @@ github.com/minio/cli v1.3.0 h1:vB0iUpmyaH54+1jJJj62Aa0qFF3xO3i0J3IcKiM6bHM=
 github.com/minio/cli v1.3.0/go.mod h1:hLsWNQy2wIf3FKFnMlH69f4RdEyn8nbRA2shaulTjGY=
 github.com/minio/cli v1.20.0 h1:OVNIt8Rg5+mpYb8siWT2gBV5hvUyFbRvBikC+Ytvf5A=
 github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
+github.com/minio/cli v1.21.0 h1:8gE8iZc0ONOhHy/T28tCsNew5f5VzWU558U9Myjfq50=
+github.com/minio/cli v1.21.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/dsync v0.0.0-20190104003057-61c41ffdeea2/go.mod h1:eLQe3mXL0h02kNpPtBJiLr1fIEIJftgXRAjncjQbxJo=
 github.com/minio/dsync v1.0.0 h1:l6pQgUPBM41idlR0UOcpAP+EYim9MCwIAUh6sQQI1gk=
 github.com/minio/dsync v1.0.0/go.mod h1:eLQe3mXL0h02kNpPtBJiLr1fIEIJftgXRAjncjQbxJo=
@@ -424,8 +430,6 @@ github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2
 github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc=
 github.com/minio/lsync v0.0.0-20190207022115-a4e43e3d0887 h1:MIpCDz3d2FR2a+FjdizuFdjsoeHuLlSkl3YNQJ55jV8=
 github.com/minio/lsync v0.0.0-20190207022115-a4e43e3d0887/go.mod h1:ni10+iSX7FO8N2rv41XM444V6w4rYO0dZo5KIkbn/YA=
-github.com/minio/lsync v0.1.0 h1:VczQvieoFS1Ayyxw6RhODkqN/UhMQXom14oJuSbR7vo=
-github.com/minio/lsync v0.1.0/go.mod h1:ni10+iSX7FO8N2rv41XM444V6w4rYO0dZo5KIkbn/YA=
 github.com/minio/lsync v1.0.1 h1:AVvILxA976xc27hstd1oR+X9PQG0sPSom1MNb1ImfUs=
 github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA=
 github.com/minio/mc v0.0.0-20190311071728-2e612b23d665/go.mod h1:7qLZXNjCD55DJ3iqe1uWoUh1MASRVd1M6wnqSdyhx7Y=
@@ -689,12 +693,8 @@ golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f h1:R423Cnkcp5JABoeemiGEPlt9tHXFfw5kvc0yqlxRPWo=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/xJ2BjNN2jWv7JoyOsMxXUU=
-golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4 h1:ydJNl0ENAG67pFbB+9tfhiL2pYqLhfoaZFw/cjLhY4A=
 golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc=
-golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -703,8 +703,6 @@ golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvx
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422 h1:QzoH/1pFpZguR8NrRHLcO6jKqfv2zpuSqZLgdm7ZmjI=
-golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -727,12 +725,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190424112056-4829fb13d2c6/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092 h1:4QSRKanuywn15aTZvI/mIDEgPQpswuFndXpOj3rKEco=
 golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE=
-golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7 h1:rTIdg5QFRR7XCaK4LCjBiPbx8j4DQRpdYMnGn/bJUEU=
-golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180603041954-1e0a3fa8ba9a/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -773,8 +767,6 @@ golang.org/x/sys v0.0.0-20190322080309-f49334f85ddc/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190429190828-d89cdac9e872/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190618155005-516e3c20635f h1:dHNZYIYdq2QuU6w73vZ/DzesPbVlZVYZTtTZmrnsbQ8=
-golang.org/x/sys v0.0.0-20190618155005-516e3c20635f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -797,14 +789,6 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190318200714-bb1270c20edf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384 h1:TFlARGu6Czu1z7q93HTxcP1P+/ZFC/IKythI5RzrnRg=
 golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59 h1:QjA/9ArTfVTLfEhClDCG7SGrZkZixxWpwNCDiwJfh88=
-golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190619181801-b76e30ffa0aa h1:OKbX3LR+DrC3Oj0rhniAcIbi43SoQW6ennRcN9MTxpE=
-golang.org/x/tools v0.0.0-20190619181801-b76e30ffa0aa/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190627203933-19ff4fff8850 h1:fA72gpMxsjJw+HAMdKku/cilEAHBQ9yEhmYkRO3wzyM=
-golang.org/x/tools v0.0.0-20190627203933-19ff4fff8850/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
-golang.org/x/tools v0.0.0-20190703212419-2214986f1668 h1:3LJOYcj2ObWSZJXX21oGIIPv5SaOoi5JkzQTWnCXRhg=
-golang.org/x/tools v0.0.0-20190703212419-2214986f1668/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 google.golang.org/api v0.0.0-20180603000442-8e296ef26005/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
 google.golang.org/api v0.0.0-20180916000451-19ff8768a5c0/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=

mint/.gitignore

@@ -0,0 +1,17 @@
+*.test
+*.jar
+src/*
+temp
+__pycache__/
+log/*
+minio.test
+bin/*
+node_modules
+# exception to the rule
+!log/.gitkeep
+!bin/.gitkeep
+*.class
+*~
+run/core/minio-dotnet/bin/*
+run/core/minio-dotnet/obj/*
+run/core/minio-dotnet/out/*

mint/Dockerfile.dev

@@ -0,0 +1,84 @@
+FROM ubuntu:16.04
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV LANG C.UTF-8
+
+ENV GOROOT /usr/local/go
+
+ENV GOPATH /usr/local
+
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+WORKDIR /mint
+
+RUN apt-get --yes update && apt-get --yes upgrade && \
+    apt-get --yes --quiet install wget jq curl dnsmasq
+
+ENV MINT_ROOT_DIR /mint
+ENV MINT_RUN_CORE_DIR $MINT_ROOT_DIR/run/core
+ENV MINT_RUN_SECURITY_DIR $MINT_ROOT_DIR/run/security
+ENV WGET "wget --quiet --no-check-certificate"
+
+COPY create-data-files.sh /mint
+RUN /mint/create-data-files.sh
+
+COPY install-packages.list /mint
+COPY preinstall.sh /mint
+RUN /mint/preinstall.sh
+
+COPY run /mint/run
+
+COPY build/awscli /mint/build/awscli
+RUN build/awscli/install.sh
+
+COPY build/aws-sdk-java /mint/build/aws-sdk-java
+RUN build/aws-sdk-java/install.sh
+
+COPY build/aws-sdk-go /mint/build/aws-sdk-go
+RUN build/aws-sdk-go/install.sh
+
+COPY build/aws-sdk-php /mint/build/aws-sdk-php
+RUN build/aws-sdk-php/install.sh
+
+COPY build/aws-sdk-ruby /mint/build/aws-sdk-ruby
+RUN build/aws-sdk-ruby/install.sh
+
+COPY build/mc /mint/build/mc
+RUN build/mc/install.sh
+
+COPY build/minio-go /mint/build/minio-go
+RUN build/minio-go/install.sh
+
+COPY build/minio-java /mint/build/minio-java
+RUN build/minio-java/install.sh
+
+COPY build/minio-js /mint/build/minio-js
+RUN build/minio-js/install.sh
+
+COPY build/minio-py /mint/build/minio-py
+RUN build/minio-py/install.sh
+
+COPY build/s3cmd /mint/build/s3cmd
+RUN build/s3cmd/install.sh
+
+COPY build/minio-dotnet/ /mint/build/minio-dotnet/
+RUN /mint/build/minio-dotnet/install.sh
+
+COPY build/security /mint/build/security
+RUN build/security/install.sh
+
+COPY build/worm /mint/build/worm
+RUN build/worm/install.sh
+
+COPY build/healthcheck /mint/build/healthcheck
+RUN build/healthcheck/install.sh
+
+COPY remove-packages.list /mint
+COPY postinstall.sh /mint
+RUN /mint/postinstall.sh
+
+COPY mint.sh /mint/mint.sh
+COPY entrypoint.sh /mint/entrypoint.sh
+
+ENTRYPOINT ["/mint/entrypoint.sh"]

mint/README.md

@@ -0,0 +1,123 @@
+# Mint [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/mint.svg?maxAge=604800)](https://hub.docker.com/r/minio/mint/)
+
+Mint is a testing framework for Minio object server, available as a docker image. It runs correctness, benchmarking and stress tests. Following are the SDKs/tools used in correctness tests.
+
+- awscli
+- aws-sdk-go
+- aws-sdk-php
+- aws-sdk-ruby
+- aws-sdk-java
+- mc
+- minio-go
+- minio-java
+- minio-js
+- minio-py
+- minio-dotnet
+- s3cmd
+- worm
+
+## Running Mint
+
+Mint is run by `docker run` command which requires Docker to be installed. For Docker installation follow the steps [here](https://docs.docker.com/engine/installation/linux/docker-ce/ubuntu/).
+
+To run Mint with Minio Play server as test target,
+
+```sh
+$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \
+             -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG -e ENABLE_HTTPS=1 minio/mint
+```
+
+After the tests are run, output is stored in `/mint/log` directory inside the container. To get these logs, use `docker cp` command. For example
+```sh
+docker cp <container-id>:/mint/log /tmp/logs
+```
+
+### Mint environment variables
+
+Below environment variables are required to be passed to the docker container. Supported environment variables:
+
+| Environment variable | Description | Example |
+|:--- |:--- |:--- |
+| `SERVER_ENDPOINT` | Endpoint of Minio server in the format `HOST:PORT`; for virtual style `IP:PORT` | `play.minio.io:9000` |
+| `ACCESS_KEY` | Access key of access `SERVER_ENDPOINT` | `Q3AM3UQ867SPQQA43P2F` |
+| `SECRET_KEY` | Secret Key of access `SERVER_ENDPOINT` | `zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG` |
+| `ENABLE_HTTPS` | (Optional) Set `1` to indicate to use HTTPS to access `SERVER_ENDPOINT`. Defaults to `0` (HTTP) | `1` |
+| `MINT_MODE` | (Optional) Set mode indicating what category of tests to be run by values `core`, `full` or `worm`.  Defaults to `core` | `full` |
+| `DOMAIN` | (Optional) Value of MINIO_DOMAIN environment variable used in Minio server | `myminio.com` |
+| `ENABLE_VIRTUAL_STYLE` | (Optional) Set `1` to indicate virtual style access . Defaults to `0` (Path style) | `1` |
+
+
+### Test virtual style access against Minio server
+
+To test Minio server virtual style access with Mint, follow these steps:
+
+- Set a domain in your Minio server using environment variable MINIO_DOMAIN. For example `export MINIO_DOMAIN=myminio.com`.
+- Start Minio server.
+- Execute Mint against Minio server (with `MINIO_DOMAIN` set to `myminio.com`) using this command
+```sh
+$ docker run -e "SERVER_ENDPOINT=192.168.86.133:9000" -e "DOMAIN=minio.com"  \
+	     -e "ACCESS_KEY=minio" -e "SECRET_KEY=minio123" -e "ENABLE_HTTPS=0" \
+	     -e "ENABLE_VIRTUAL_STYLE=1" minio/mint
+```
+
+### Mint log format
+
+All test logs are stored in `/mint/log/log.json` as multiple JSON document.  Below is the JSON format for every entry in the log file.
+
+| JSON field | Type | Description | Example |
+|:--- |:--- |:--- |:--- |
+| `name` | _string_ | Testing tool/SDK name | `"aws-sdk-php"` |
+| `function` | _string_ | Test function name | `"getBucketLocation ( array $params = [] )"` |
+| `args` | _object_ | (Optional) Key/Value map of arguments passed to test function | `{"Bucket":"aws-sdk-php-bucket-20341"}` |
+| `duration` | _int_ | Time taken in milliseconds to run the test | `384` |
+| `status` | _string_ | one of `PASS`, `FAIL` or `NA` | `"PASS"` |
+| `alert` | _string_ | (Optional) Alert message indicating test failure | `"I/O error on create file"` |
+| `message` | _string_ | (Optional) Any log message | `"validating checksum of downloaded object"` |
+| `error` | _string_ | Detailed error message including stack trace on status `FAIL` | `"Error executing \"CompleteMultipartUpload\" on ...` |
+
+## For Developers
+
+### Running Mint development code
+
+After making changes to Mint source code a local docker image can be built/run by
+
+```sh
+$ docker build -t minio/mint . -f Dockerfile.dev
+$ docker run -e SERVER_ENDPOINT=play.minio.io:9000 -e ACCESS_KEY=Q3AM3UQ867SPQQA43P2F \
+             -e SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \
+             -e ENABLE_HTTPS=1 -e MINT_MODE=full minio/mint:latest
+```
+In case of Worm Mode, start your server with configuration `MINT_MODE` set to `worm`.
+Build/Run of local docker image for Worm mode, is to be tested against your server configuration, by
+
+```
+### Adding tests with new tool/SDK
+
+Below are the steps need to be followed
+
+* Create new app directory under [build](https://github.com/minio/mint/tree/master/build) and [run/core](https://github.com/minio/mint/tree/master/run/core) directories.
+* Create `install.sh` which does installation of required tool/SDK under app directory.
+* Any build and install time dependencies should be added to [install-packages.list](https://github.com/minio/mint/blob/master/install-packages.list).
+* Build time dependencies should be added to [remove-packages.list](https://github.com/minio/mint/blob/master/remove-packages.list) for removal to have clean Mint docker image.
+* Add `run.sh` in app directory under `run/core` which execute actual tests.
+
+#### Test data
+
+Tests may use pre-created data set to perform various object operations on Minio server.  Below data files are available under `/mint/data` directory.
+
+| File name |  Size |
+|:--- |:--- |
+| datafile-0-b | 0B |
+| datafile-1-b | 1B |
+| datafile-1-kB |1KiB |
+| datafile-10-kB |10KiB |
+| datafile-33-kB |33KiB |
+| datafile-100-kB |100KiB |
+| datafile-1-MB |1MiB |
+| datafile-1.03-MB |1.03MiB |
+| datafile-5-MB |5MiB |
+| datafile-6-MB |6MiB |
+| datafile-10-MB |10MiB |
+| datafile-11-MB |11MiB |
+| datafile-65-MB |65MiB |
+| datafile-129-MB |129MiB |

mint/build/aws-sdk-go/install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+#
+#  Mint (C) 2017 Minio, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-go"
+GO111MODULE=on go build -o "$test_run_dir/aws-sdk-go" "$test_run_dir/quick-tests.go"

mint/build/aws-sdk-java/build.xml

@@ -0,0 +1,58 @@
+<project xmlns:ivy="antlib:org.apache.ivy.ant" name="aws-sdk-java-tests" default="run">
+    <property name="ivy.install.version" value="2.1.0-rc2" />
+    <condition property="ivy.home" value="${env.IVY_HOME}">
+      <isset property="env.IVY_HOME" />
+    </condition>
+    <property name="ivy.home" value="${user.home}/.ant" />
+    <property name="ivy.jar.dir" value="${ivy.home}/lib" />
+    <property name="ivy.jar.file" value="${ivy.jar.dir}/ivy.jar" />
+
+    <target name="download-ivy" unless="offline">
+        <mkdir dir="${ivy.jar.dir}"/>
+        <get src="https://repo1.maven.org/maven2/org/apache/ivy/ivy/${ivy.install.version}/ivy-${ivy.install.version}.jar" 
+             dest="${ivy.jar.file}" usetimestamp="true"/>
+    </target>
+
+    <target name="init-ivy" depends="download-ivy">
+        <path id="ivy.lib.path">
+            <fileset dir="${ivy.jar.dir}" includes="*.jar"/>
+
+        </path>
+        <taskdef resource="org/apache/ivy/ant/antlib.xml"
+                 uri="antlib:org.apache.ivy.ant" classpathref="ivy.lib.path"/>
+    </target>
+    <target name="resolve" description="--> retrieve dependencies with ivy">
+        <ivy:retrieve />
+    </target>
+
+    <target name="clean">
+        <delete dir="build"/>
+    </target>
+
+    <path id="aws-s3-sdk-deps">
+        <fileset dir="lib">
+            <include name="*.jar"/>
+        </fileset>
+    </path>
+
+    <target name="compile">
+        <mkdir dir="build/classes"/>
+        <javac srcdir="src" destdir="build/classes">
+            <classpath refid="aws-s3-sdk-deps" />
+        </javac>
+    </target>
+
+    <target name="jar">
+        <mkdir dir="build/jar"/>
+        <jar destfile="build/jar/FunctionalTests.jar" basedir="build/classes">
+	    <archives>
+	        <zips>
+	            <fileset dir="lib/" includes="*.jar"/>
+	         </zips>
+	    </archives>
+            <manifest>
+                <attribute name="Main-Class" value="io.minio.awssdk.tests.FunctionalTests"/>
+            </manifest>
+        </jar>
+    </target>
+</project>

mint/build/aws-sdk-java/install.sh

@@ -0,0 +1,30 @@
+#!/bin/bash -e
+#
+#  Mint (C) 2017 Minio, Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+#
+
+test_run_dir="$MINT_RUN_CORE_DIR/aws-sdk-java"
+
+cd "$(dirname "$(realpath "$0")")"
+
+ant init-ivy && \
+    ant resolve && \
+    ant compile && \
+    ant jar
+
+cp build/jar/FunctionalTests.jar "$test_run_dir/"
+
+rm -rf lib/ build/
+

mint/build/aws-sdk-java/ivy.xml

@@ -0,0 +1,6 @@
+<ivy-module version="2.0">
+    <info organisation="org.apache" module="aws-sdk-java-tests"/>
+    <dependencies>
+	<dependency org="com.amazonaws" name="aws-java-sdk-s3" rev="1.11.289"/>
+    </dependencies>
+</ivy-module>