Add missing error check in cache GetObjectNInfo (#8889)

MakeVol shouldn't be called in heal bucket
when bucket doesn't really exist in quorum.

.jshintrc

@@ -1,4 +0,0 @@
-{
-  "asi": true,
-  "esnext": true   
-}

.mention-bot

@@ -1,4 +0,0 @@
-{
-    "numFilesToCheck": 10,
-    "requiredOrgs": ["minio"]
-}

CREDITS

@@ -11366,28 +11366,28 @@ SOFTWARE.
 github.com/klauspost/readahead
 https://github.com/klauspost/readahead
 ----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) 2015 Klaus Post
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
+The MIT License (MIT)
+
+Copyright (c) 2015 Klaus Post
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 
 ================================================================
 
@@ -12375,6 +12375,214 @@ https://github.com/minio/minio-go/v6
 
 ================================================================
 
+github.com/minio/minio-go/v6
+https://github.com/minio/minio-go/v6
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 github.com/minio/parquet-go
 https://github.com/minio/parquet-go
 ----------------------------------------------------------------
@@ -16208,7 +16416,7 @@ github.com/pascaldekloe/goe
 https://github.com/pascaldekloe/goe
 ----------------------------------------------------------------
 To the extent possible under law, Pascal S. de Kloe has waived all
-copyright and related or neighboring rights to Go Enterprise. This
+copyright and related or neighboring rights to Go Enterprice. This
 work is published from The Netherlands.
 
 https://creativecommons.org/publicdomain/zero/1.0/legalcode
@@ -16312,36 +16520,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-github.com/pkg/profile
-https://github.com/pkg/profile
-----------------------------------------------------------------
-Copyright (c) 2013 Dave Cheney. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-   * Redistributions of source code must retain the above copyright
-notice, this list of conditions and the following disclaimer.
-   * Redistributions in binary form must reproduce the above
-copyright notice, this list of conditions and the following disclaimer
-in the documentation and/or other materials provided with the
-distribution.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-================================================================
-
 github.com/pmezard/go-difflib
 https://github.com/pmezard/go-difflib
 ----------------------------------------------------------------

Dockerfile

@@ -5,7 +5,6 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 ENV GOPATH /go
 ENV CGO_ENABLED 0
 ENV GO111MODULE on
-ENV GOPROXY https://proxy.golang.org
 
 RUN  \
      apk add --no-cache git && \

NOTICE

@@ -1,4 +1,4 @@
-MinIO Cloud Storage, (C) 2014,2015 MinIO, Inc.
+MinIO Cloud Storage, (C) 2014-2020 MinIO, Inc.
 
 This product includes software developed at MinIO, Inc.
 (https://min.io/).
@@ -6,4 +6,4 @@ This product includes software developed at MinIO, Inc.
 The MinIO project contains unmodified/modified subcomponents too with
 separate copyright notices and license terms. Your use of the source
 code for these subcomponents is subject to the terms and conditions
-of the following licenses.
+of Apache License Version 2.0

README.md

@@ -21,7 +21,7 @@ docker run -p 9000:9000 minio/minio:edge server /data
 > NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
 
 ## macOS
-### Homebrew
+### Homebrew (recommended)
 Install minio packages using [Homebrew](http://brew.sh/)
 ```sh
 brew install minio/stable/minio
@@ -74,7 +74,7 @@ minio.exe server D:\Photos
 
 ## FreeBSD
 ### Port
-Install minio packages using [pkg](https://github.com/freebsd/pkg)
+Install minio packages using [pkg](https://github.com/freebsd/pkg), MinIO doesn't officially build FreeBSD binaries but is maintained by FreeBSD upstream [here](https://www.freshports.org/www/minio).
 
 ```sh
 pkg install minio

buildscripts/benchcmp.sh

@@ -1,67 +0,0 @@
-#!/bin/bash          
-
-# usage: ./benchcmp.sh <commit-sha1> <commit-sha2>
-# Exit on any non zero return value on execution of a command. 
-set -e 
-
-# path of benchcmp. 
-benchcmp=${GOPATH}/bin/benchcmp
-
-# function which runs the benchmark comparison. 
-RunBenchCmp () {
-	# Path for storing output of benchmark at commit 1. 
-	commit1Bench=/tmp/minio-$1.bench
-	# Path for storing output of benchmark at commit 2. 
-	commit2Bench=/tmp/minio-$2.bench
-	# switch to commit $1. 
-	git checkout $1
-	# Check if the benchmark results for given commit 1 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit1Bench  ]]
-	then
-
-		echo "Running benchmarks at $1"
-		go test -run=NONE -bench=. | tee $commit1Bench 
-	fi
-	# get back to the commit from which it was started. 
-	git checkout -
-	echo "Checking into commit $2"
-	# switch to commit $2 
-	git checkout $2
-	# Check if the benchmark results for given commit 2 already exists. 
-	# Benchmarks are time/resource consuming operations, run only if the the results doesn't exist. 
-	if [[ ! -f $commit2Bench  ]]
-	then
-	# Running benchmarks at $2.
-		echo "Running benchmarks at $2"
-		go test -run=NONE -bench=. | tee $commit2Bench 
-	fi
-	
-	# get back to the commit from which it was started. 
-	git checkout - 
-	# Comparing the benchmarks.
-	echo "Running benchmark comparison between $1 and $2 ..."
-	$benchcmp $commit1Bench $commit2Bench 
-	echo "Done."
-}
-
-# check if 2 commit SHA's of snapshots of code for which benchmp has to be done is provided. 
-if [ ! $# -eq 2 ]
-then
-	# exit if commit SHA's are not provided. 
-	echo $#
-	echo "Need Commit SHA's of 2 snapshots to be supplied to run benchmark comparison."
-	exit 1
-fi
-
-# check if benchcmp exists. 
-if [[ -x "$benchcmp" ]]
-then
-	RunBenchCmp $1 $2
-else
-	# install benchcmp if doesnt't exist. 
-	echo "fetching Benchcmp..."
-	go get -u golang.org/x/tools/cmd/benchcmp
-	echo "Done."
-	RunBenchCmp $1 $2
-fi

cmd/admin-handlers.go

@@ -302,13 +302,13 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	dataUsageInfoJSON, err := json.Marshal(dataUsageInfo)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 

cmd/admin-handlers_test.go

@@ -109,7 +109,7 @@ func initTestXLObjLayer() (ObjectLayer, []string, error) {
 		return nil, nil, err
 	}
 	endpoints := mustGetNewEndpoints(xlDirs...)
-	format, err := waitForFormatXL(true, endpoints, 1, 16, "")
+	format, err := waitForFormatXL(true, endpoints, 1, 1, 16, "")
 	if err != nil {
 		removeRoots(xlDirs)
 		return nil, nil, err

cmd/api-errors.go

@@ -34,7 +34,9 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 // APIError structure
@@ -149,6 +151,7 @@ const (
 	ErrPastObjectLockRetainDate
 	ErrUnknownWORMModeDirective
 	ErrObjectLockInvalidHeaders
+	ErrInvalidTagDirective
 	// Add new error codes here.
 
 	// SSE-S3 related API errors
@@ -829,6 +832,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your metadata headers exceed the maximum allowed metadata size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidTagDirective: {
+		Code:           "InvalidArgument",
+		Description:    "Unknown tag directive.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidEncryptionMethod: {
 		Code:           "InvalidRequest",
 		Description:    "The encryption method specified is not supported",
@@ -1611,14 +1619,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
 		apiErr = ErrSlowDown
-	case errInvalidRetentionDate:
+	case objectlock.ErrInvalidRetentionDate:
 		apiErr = ErrInvalidRetentionDate
-	case errPastObjectLockRetainDate:
+	case objectlock.ErrPastObjectLockRetainDate:
 		apiErr = ErrPastObjectLockRetainDate
-	case errUnknownWORMModeDirective:
+	case objectlock.ErrUnknownWORMModeDirective:
 		apiErr = ErrUnknownWORMModeDirective
-	case errObjectLockInvalidHeaders:
+	case objectlock.ErrObjectLockInvalidHeaders:
 		apiErr = ErrObjectLockInvalidHeaders
+	case objectlock.ErrMalformedXML:
+		apiErr = ErrMalformedXML
 	}
 
 	// Compression errors
@@ -1777,6 +1787,12 @@ func toAPIError(ctx context.Context, err error) APIError {
 		// their internal error types. This code is only
 		// useful with gateway implementations.
 		switch e := err.(type) {
+		case tagging.Error:
+			apiErr = APIError{
+				Code:           "InvalidTag",
+				Description:    e.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case policy.Error:
 			apiErr = APIError{
 				Code:           "MalformedPolicy",

cmd/api-headers.go

@@ -22,6 +22,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
+	"net/url"
 	"strconv"
 	"time"
 
@@ -95,6 +96,14 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
 		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
 	}
+
+	// Set tag count if object has tags
+	tags, _ := url.ParseQuery(objInfo.UserTags)
+	tagCount := len(tags)
+	if tagCount != 0 {
+		w.Header().Set(xhttp.AmzTagCount, strconv.Itoa(tagCount))
+	}
+
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
 		if HasPrefix(k, ReservedMetadataPrefix) {

cmd/api-router.go

@@ -105,26 +105,31 @@ func registerAPIRouter(router *mux.Router, encryptionEnabled, allowSSEKMS bool)
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler))).Queries("acl", "")
-		// GetObjectTagging - this is a dummy call.
+		// GetObjectTagging
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler))).Queries("tagging", "")
+		// PutObjectTagging
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler))).Queries("tagging", "")
+		// DeleteObjectTagging
+		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler))).Queries("tagging", "")
 		// SelectObjectContent
 		bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler))).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectretention", httpTraceHdrs(api.GetObjectRetentionHandler))).Queries("retention", "")
+		// GetObjectLegalHold
+		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
 		// GetObject
 		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler)))
 		// CopyObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler)))
 		// PutObjectRetention
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectretention", httpTraceHdrs(api.PutObjectRetentionHandler))).Queries("retention", "")
+		// PutObjectLegalHold
+		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
+
 		// PutObject
 		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler)))
 		// DeleteObject
 		bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler)))
-		// PutObjectLegalHold
-		bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(collectAPIStats("putobjectlegalhold", httpTraceHdrs(api.PutObjectLegalHoldHandler))).Queries("legal-hold", "")
-		// GetObjectLegalHold
-		bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(collectAPIStats("getobjectlegalhold", httpTraceHdrs(api.GetObjectLegalHoldHandler))).Queries("legal-hold", "")
 
 		/// Bucket operations
 		// GetBucketLocation

cmd/background-heal-ops.go

@@ -75,7 +75,7 @@ func (h *healRoutine) run() {
 
 			var res madmin.HealResultItem
 			var err error
-			bucket, object := urlPath2BucketObjectName(task.path)
+			bucket, object := path2BucketObject(task.path)
 			switch {
 			case bucket == "" && object == "":
 				res, err = bgHealDiskFormat(ctx, task.opts)

cmd/bucket-handlers.go

@@ -40,6 +40,7 @@ import (
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
+	"github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -533,7 +534,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	if globalDNSConfig != nil {
-		if _, err := globalDNSConfig.Get(bucket); err != nil {
+		sr, err := globalDNSConfig.Get(bucket)
+		if err != nil {
 			if err == dns.ErrNoEntriesFound {
 				// Proceed to creating a bucket.
 				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
@@ -566,7 +568,15 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			return
 
 		}
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBucketAlreadyOwnedByYou), r.URL, guessIsBrowserReq(r))
+		apiErr := ErrBucketAlreadyExists
+		if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
+			apiErr = ErrBucketAlreadyOwnedByYou
+		}
+		// No IPs seem to intersect, this means that bucket exists but has
+		// different IP addresses perhaps from a different deployment.
+		// bucket names are globally unique in federation at a given
+		// path prefix, name collision is not allowed. Return appropriate error.
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
@@ -577,14 +587,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if objectLockEnabled {
+	if objectLockEnabled && !globalIsGateway {
 		configFile := path.Join(bucketConfigPrefix, bucket, bucketObjectLockEnabledConfigFile)
 		if err = saveConfig(ctx, objectAPI, configFile, []byte(bucketObjectLockEnabledConfig)); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		globalBucketObjectLockConfig.Set(bucket, Retention{})
-		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, Retention{})
+		globalBucketObjectLockConfig.Set(bucket, objectlock.Retention{})
+		globalNotificationSys.PutBucketObjectLockConfig(ctx, bucket, objectlock.Retention{})
 	}
 
 	// Make sure to add Location information here only for bucket
@@ -1005,7 +1015,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	config, err := parseObjectLockConfig(r.Body)
+	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		apiErr.Description = err.Error()
@@ -1099,7 +1109,7 @@ func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWri
 			return
 		}
 
-		if configData, err = xml.Marshal(newObjectLockConfig()); err != nil {
+		if configData, err = xml.Marshal(objectlock.NewObjectLockConfig()); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}

cmd/config-current.go

@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/logger/target/http"
 	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 func initHelp() {
@@ -469,6 +470,20 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 
 	envHelp := config.HelpKVS{}
 	if envOnly {
+		// Only for multiple targets, make sure
+		// to list the ENV, for regular k/v EnableKey is
+		// implicit, for ENVs we cannot make it implicit.
+		if subSysHelp.MultipleTargets {
+			envK := config.EnvPrefix + strings.Join([]string{
+				strings.ToTitle(subSys), strings.ToTitle(madmin.EnableKey),
+			}, config.EnvWordDelimiter)
+			envHelp = append(envHelp, config.HelpKV{
+				Key:         envK,
+				Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
+				Optional:    false,
+				Type:        "on|off",
+			})
+		}
 		for _, hkv := range h {
 			envK := config.EnvPrefix + strings.Join([]string{
 				strings.ToTitle(subSys), strings.ToTitle(hkv.Key),

cmd/config/identity/openid/jwt.go

@@ -251,6 +251,7 @@ func parseDiscoveryDoc(u *xnet.URL, transport *http.Transport, closeRespFn func(
 	}
 	resp, err := clnt.Do(req)
 	if err != nil {
+		clnt.CloseIdleConnections()
 		return d, err
 	}
 	defer closeRespFn(resp.Body)

cmd/data-usage.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"os"
+	"path/filepath"
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
@@ -48,63 +50,58 @@ func runDataUsageInfoUpdateRoutine() {
 		break
 	}
 
-	ctx := context.Background()
-
-	switch v := objAPI.(type) {
-	case *xlZones:
-		runDataUsageInfoForXLZones(ctx, v, GlobalServiceDoneCh)
-	case *FSObjects:
-		runDataUsageInfoForFS(ctx, v, GlobalServiceDoneCh)
-	default:
-		return
-	}
+	runDataUsageInfo(context.Background(), objAPI, GlobalServiceDoneCh)
 }
 
-func runDataUsageInfoForFS(ctx context.Context, fsObj *FSObjects, endCh <-chan struct{}) {
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
-	for {
-		// Get data usage info of the FS Object
-		usageInfo := fsObj.crawlAndGetDataUsageInfo(ctx, endCh)
-		// Save the data usage in the disk
-		err := storeDataUsageInBackend(ctx, fsObj, usageInfo)
-		if err != nil {
-			logger.LogIf(ctx, err)
-		}
-		select {
-		case <-endCh:
-			return
-		// Wait until the next crawl interval
-		case <-t.C:
-		}
+// timeToNextCrawl returns the duration until next crawl should occur
+// this is validated by verifying the LastUpdate time.
+func timeToCrawl(ctx context.Context, objAPI ObjectLayer) time.Duration {
+	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objAPI)
+	if err != nil {
+		// Upon an error wait for like 10
+		// seconds to start the crawler.
+		return 10 * time.Second
 	}
+	// File indeed doesn't exist when LastUpdate is zero
+	// so we have never crawled, start crawl right away.
+	if dataUsageInfo.LastUpdate.IsZero() {
+		return 1 * time.Second
+	}
+	waitDuration := dataUsageInfo.LastUpdate.Sub(UTCNow())
+	if waitDuration > dataUsageCrawlInterval {
+		// Waited long enough start crawl in a 1 second
+		return 1 * time.Second
+	}
+	// No crawling needed, ask the routine to wait until
+	// the daily interval 12hrs - delta between last update
+	// with current time.
+	return dataUsageCrawlInterval - waitDuration
 }
 
-func runDataUsageInfoForXLZones(ctx context.Context, z *xlZones, endCh <-chan struct{}) {
-	locker := z.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
+func runDataUsageInfo(ctx context.Context, objAPI ObjectLayer, endCh <-chan struct{}) {
+	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "leader-data-usage-info")
 	for {
 		err := locker.GetLock(newDynamicTimeout(time.Millisecond, time.Millisecond))
 		if err != nil {
 			time.Sleep(5 * time.Minute)
 			continue
 		}
-		// Break without locking
+		// Break without unlocking, this node will acquire
+		// data usage calculator role for its lifetime.
 		break
 	}
 
-	t := time.NewTicker(dataUsageCrawlInterval)
-	defer t.Stop()
 	for {
-		usageInfo := z.crawlAndGetDataUsage(ctx, endCh)
-		err := storeDataUsageInBackend(ctx, z, usageInfo)
-		if err != nil {
-			logger.LogIf(ctx, err)
-		}
+		wait := timeToCrawl(ctx, objAPI)
 		select {
 		case <-endCh:
 			locker.Unlock()
 			return
-		case <-t.C:
+		case <-time.NewTimer(wait).C:
+			// Crawl only when no previous crawl has occurred,
+			// or its been too long since last crawl.
+			err := storeDataUsageInBackend(ctx, objAPI, objAPI.CrawlAndGetDataUsage(ctx, endCh))
+			logger.LogIf(ctx, err)
 		}
 	}
 }
@@ -130,7 +127,10 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	err := objAPI.GetObject(ctx, minioMetaBackgroundOpsBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
 	if err != nil {
-		return DataUsageInfo{}, nil
+		if isErrObjectNotFound(err) {
+			return DataUsageInfo{}, nil
+		}
+		return DataUsageInfo{}, toObjectErr(err, minioMetaBackgroundOpsBucket, dataUsageObjName)
 	}
 
 	var dataUsageInfo DataUsageInfo
@@ -142,3 +142,85 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 
 	return dataUsageInfo, nil
 }
+
+// Item represents each file while walking.
+type Item struct {
+	Path string
+	Typ  os.FileMode
+}
+
+type getSizeFn func(item Item) (int64, error)
+type activeIOFn func() error
+
+func updateUsage(basePath string, endCh <-chan struct{}, waitForLowActiveIO activeIOFn, getSize getSizeFn) DataUsageInfo {
+	var dataUsageInfo = DataUsageInfo{
+		BucketsSizes:          make(map[string]uint64),
+		ObjectsSizesHistogram: make(map[string]uint64),
+	}
+
+	itemCh := make(chan Item)
+	skipCh := make(chan error)
+	defer close(skipCh)
+
+	go func() {
+		defer close(itemCh)
+		fastWalk(basePath, func(path string, typ os.FileMode) error {
+			if err := waitForLowActiveIO(); err != nil {
+				return filepath.SkipDir
+			}
+
+			select {
+			case <-endCh:
+				return filepath.SkipDir
+			case itemCh <- Item{path, typ}:
+			}
+			return <-skipCh
+		})
+	}()
+
+	for {
+		select {
+		case <-endCh:
+			return dataUsageInfo
+		case item, ok := <-itemCh:
+			if !ok {
+				return dataUsageInfo
+			}
+
+			bucket, entry := path2BucketObjectWithBasePath(basePath, item.Path)
+			if bucket == "" {
+				skipCh <- nil
+				continue
+			}
+
+			if isReservedOrInvalidBucket(bucket, false) {
+				skipCh <- filepath.SkipDir
+				continue
+			}
+
+			if entry == "" && item.Typ&os.ModeDir != 0 {
+				dataUsageInfo.BucketsCount++
+				dataUsageInfo.BucketsSizes[bucket] = 0
+				skipCh <- nil
+				continue
+			}
+
+			if item.Typ&os.ModeDir != 0 {
+				skipCh <- nil
+				continue
+			}
+
+			size, err := getSize(item)
+			if err != nil {
+				skipCh <- errSkipFile
+				continue
+			}
+
+			dataUsageInfo.ObjectsCount++
+			dataUsageInfo.ObjectsTotalSize += uint64(size)
+			dataUsageInfo.BucketsSizes[bucket] += uint64(size)
+			dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
+			skipCh <- nil
+		}
+	}
+}

cmd/disk-cache.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	"github.com/minio/minio/pkg/wildcard"
 )
@@ -251,7 +252,13 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		c.cacheStats.incMiss()
 		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
-
+	// skip cache for objects with locks
+	objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
+	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
+		c.cacheStats.incMiss()
+		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	}
 	if cacheErr == nil {
 		// if ETag matches for stale cache entry, serve from cache
 		if cacheReader.ObjInfo.ETag == objInfo.ETag {
@@ -277,6 +284,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	}
 
 	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	if bkErr != nil {
+		return bkReader, bkErr
+	}
 	// Record if cache has a hit that was invalidated by ETag verification
 	if cacheErr == nil {
 		bkReader.ObjInfo.CacheLookupStatus = CacheHit
@@ -596,8 +606,9 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	}
 
 	// skip cache for objects with locks
-	objRetention := getObjectRetentionMeta(opts.UserDefined)
-	if objRetention.Mode == Governance || objRetention.Mode == Compliance {
+	objRetention := objectlock.GetObjectRetentionMeta(opts.UserDefined)
+	legalHold := objectlock.GetObjectLegalHoldMeta(opts.UserDefined)
+	if objRetention.Mode != objectlock.Invalid || legalHold.Status != "" {
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}

cmd/dummy-handlers.go

@@ -22,26 +22,13 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 // Data types used for returning dummy tagging XML.
 // These variables shouldn't be used elsewhere.
 // They are only defined to be used in this file alone.
 
-type tagging struct {
-	XMLName xml.Name `xml:"Tagging"`
-	TagSet  tagSet   `xml:"TagSet"`
-}
-
-type tagSet struct {
-	Tag []tagElem `xml:"Tag"`
-}
-
-type tagElem struct {
-	Key   string `xml:"Key"`
-	Value string `xml:"Value"`
-}
-
 // GetBucketWebsite  - GET bucket website, a dummy api
 func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) {
 	writeSuccessResponseHeadersOnly(w)
@@ -171,47 +158,8 @@ func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	tags := &tagging{}
-	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
-
-	if err := xml.NewEncoder(w).Encode(tags); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	w.(http.Flusher).Flush()
-}
-
-// GetObjectTaggingHandler - GET object tagging, a dummy api
-func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetObjectTagging")
-
-	vars := mux.Vars(r)
-	bucket := vars["bucket"]
-	object := vars["object"]
-
-	objAPI := api.ObjectAPI()
-	if objAPI == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Allow getObjectTagging if policy action is set, since this is a dummy call
-	// we are simply re-purposing the bucketPolicyAction.
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	// Validate if object exists, before proceeding further...
-	_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
-	}
-
-	tags := &tagging{}
-	tags.TagSet.Tag = append(tags.TagSet.Tag, tagElem{})
+	tags := &tagging.Tagging{}
+	tags.TagSet.Tags = append(tags.TagSet.Tags, tagging.Tag{})
 
 	if err := xml.NewEncoder(w).Encode(tags); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))

cmd/fastwalk.go

@@ -17,14 +17,7 @@ import (
 	"sync"
 )
 
-// ErrTraverseLink is used as a return value from WalkFuncs to indicate that the
-// symlink named in the call may be traversed.
-var ErrTraverseLink = errors.New("fastwalk: traverse symlink, assuming target is a directory")
-
-// ErrSkipFiles is a used as a return value from WalkFuncs to indicate that the
-// callback should not be called for any other files in the current directory.
-// Child directories will still be traversed.
-var ErrSkipFiles = errors.New("fastwalk: skip remaining files in directory")
+var errSkipFile = errors.New("fastwalk: skip this file")
 
 // Walk is a faster implementation of filepath.Walk.
 //
@@ -161,25 +154,32 @@ func (w *walker) enqueue(it walkItem) {
 	}
 }
 
+var stringsBuilderPool = sync.Pool{
+	New: func() interface{} {
+		return &strings.Builder{}
+	},
+}
+
 func (w *walker) onDirEnt(dirName, baseName string, typ os.FileMode) error {
-	joined := dirName + string(os.PathSeparator) + baseName
+	builder := stringsBuilderPool.Get().(*strings.Builder)
+	defer func() {
+		builder.Reset()
+		stringsBuilderPool.Put(builder)
+	}()
+
+	builder.WriteString(dirName)
+	if !strings.HasSuffix(dirName, SlashSeparator) {
+		builder.WriteString(SlashSeparator)
+	}
+	builder.WriteString(baseName)
 	if typ == os.ModeDir {
-		w.enqueue(walkItem{dir: joined})
+		w.enqueue(walkItem{dir: builder.String()})
 		return nil
 	}
 
-	err := w.fn(joined, typ)
-	if typ == os.ModeSymlink {
-		if err == ErrTraverseLink {
-			// Set callbackDone so we don't call it twice for both the
-			// symlink-as-symlink and the symlink-as-directory later:
-			w.enqueue(walkItem{dir: joined, callbackDone: true})
-			return nil
-		}
-		if err == filepath.SkipDir {
-			// Permit SkipDir on symlinks too.
-			return nil
-		}
+	err := w.fn(builder.String(), typ)
+	if err == filepath.SkipDir || err == errSkipFile {
+		return nil
 	}
 	return err
 }
@@ -189,22 +189,13 @@ func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode)
 	if err != nil {
 		return err
 	}
-	skipFiles := false
 	for _, fi := range fis {
 		var mode os.FileMode
 		if strings.HasSuffix(fi, SlashSeparator) {
 			mode |= os.ModeDir
 		}
 
-		if mode == 0 && skipFiles {
-			continue
-		}
-
-		if err := fn(dirName, fi, mode); err != nil {
-			if err == ErrSkipFiles {
-				skipFiles = true
-				continue
-			}
+		if err = fn(dirName, fi, mode); err != nil {
 			return err
 		}
 	}
@@ -214,7 +205,7 @@ func readDirFn(dirName string, fn func(dirName, entName string, typ os.FileMode)
 func (w *walker) walk(root string, runUserCallback bool) error {
 	if runUserCallback {
 		err := w.fn(root, os.ModeDir)
-		if err == filepath.SkipDir {
+		if err == filepath.SkipDir || err == errSkipFile {
 			return nil
 		}
 		if err != nil {

cmd/fs-v1-metadata.go

@@ -182,9 +182,14 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
 			objInfo.Expires = t.UTC()
 		}
 	}
+
+	// Add user tags to the object info
+	objInfo.UserTags = m.Meta[xhttp.AmzObjectTagging]
+
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
+	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(m.Meta)
 
 	// All the parts per object.

cmd/fs-v1.go

@@ -27,7 +27,6 @@ import (
 	"os"
 	"os/user"
 	"path"
-	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -37,6 +36,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/lock"
@@ -44,6 +44,7 @@ import (
 	"github.com/minio/minio/pkg/mimedb"
 	"github.com/minio/minio/pkg/mountinfo"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 // Default etag is used for pre-existing objects.
@@ -109,8 +110,7 @@ func initMetaVolumeFS(fsPath, fsUUID string) error {
 		return err
 	}
 
-	metaStatsPath := pathJoin(fsPath, minioMetaBackgroundOpsBucket, fsUUID)
-	if err := os.MkdirAll(metaStatsPath, 0777); err != nil {
+	if err := os.MkdirAll(pathJoin(fsPath, minioMetaBackgroundOpsBucket), 0777); err != nil {
 		return err
 	}
 
@@ -227,90 +227,30 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
 }
 
 func (fs *FSObjects) waitForLowActiveIO() error {
-	t := time.NewTicker(lowActiveIOWaitTick)
-	defer t.Stop()
-	for {
-		if atomic.LoadInt64(&fs.activeIOCount) >= fs.maxActiveIOCount {
-			select {
-			case <-GlobalServiceDoneCh:
-				return errors.New("forced exit")
-			case <-t.C:
-				continue
-			}
+	for atomic.LoadInt64(&fs.activeIOCount) >= fs.maxActiveIOCount {
+		select {
+		case <-GlobalServiceDoneCh:
+			return errors.New("forced exit")
+		case <-time.NewTimer(lowActiveIOWaitTick).C:
+			continue
 		}
-		break
 	}
 
 	return nil
 
 }
 
-// crawlAndGetDataUsageInfo returns data usage stats of the current FS deployment
-func (fs *FSObjects) crawlAndGetDataUsageInfo(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
-
-	var dataUsageInfoMu sync.Mutex
-	var dataUsageInfo = DataUsageInfo{
-		BucketsSizes:          make(map[string]uint64),
-		ObjectsSizesHistogram: make(map[string]uint64),
-	}
-
-	walkFn := func(origPath string, typ os.FileMode) error {
-
-		select {
-		case <-GlobalServiceDoneCh:
-			return filepath.SkipDir
-		default:
-		}
-
-		if err := fs.waitForLowActiveIO(); err != nil {
-			return filepath.SkipDir
-		}
-
-		path := strings.TrimPrefix(origPath, fs.fsPath)
-		path = strings.TrimPrefix(path, SlashSeparator)
-
-		splits := splitN(path, SlashSeparator, 2)
-		bucket := splits[0]
-		prefix := splits[1]
-
-		if bucket == "" {
-			return nil
-		}
-
-		if isReservedOrInvalidBucket(bucket, false) {
-			return filepath.SkipDir
-		}
-
-		if prefix == "" {
-			dataUsageInfoMu.Lock()
-			dataUsageInfo.BucketsCount++
-			dataUsageInfo.BucketsSizes[bucket] = 0
-			dataUsageInfoMu.Unlock()
-			return nil
-		}
-
-		if typ&os.ModeDir != 0 {
-			return nil
-		}
-
-		// Get file size
-		fi, err := os.Stat(origPath)
+// CrawlAndGetDataUsage returns data usage stats of the current FS deployment
+func (fs *FSObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+	dataUsageInfo := updateUsage(fs.fsPath, endCh, fs.waitForLowActiveIO, func(item Item) (int64, error) {
+		// Get file size, symlinks which cannot bex
+		// followed are automatically filtered by fastwalk.
+		fi, err := os.Stat(item.Path)
 		if err != nil {
-			return nil
+			return 0, errSkipFile
 		}
-		size := fi.Size()
-
-		dataUsageInfoMu.Lock()
-		dataUsageInfo.ObjectsCount++
-		dataUsageInfo.ObjectsTotalSize += uint64(size)
-		dataUsageInfo.BucketsSizes[bucket] += uint64(size)
-		dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(size))]++
-		dataUsageInfoMu.Unlock()
-
-		return nil
-	}
-
-	fastWalk(fs.fsPath, walkFn)
+		return fi.Size(), nil
+	})
 
 	dataUsageInfo.LastUpdate = UTCNow()
 	atomic.StoreUint64(&fs.totalUsed, dataUsageInfo.ObjectsTotalSize)
@@ -1235,6 +1175,58 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
 		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
 }
 
+// GetObjectTag - get object tags from an existing object
+func (fs *FSObjects) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	oi, err := fs.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+
+	tags, err := tagging.FromString(oi.UserTags)
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+
+	return tags, nil
+}
+
+// PutObjectTag - replace or add tags to an existing object
+func (fs *FSObjects) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+	fsMeta := fsMetaV1{}
+	wlk, err := fs.rwPool.Write(fsMetaPath)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return toObjectErr(err, bucket, object)
+	}
+	// This close will allow for locks to be synchronized on `fs.json`.
+	defer wlk.Close()
+
+	// Read objects' metadata in `fs.json`.
+	if _, err = fsMeta.ReadFrom(ctx, wlk); err != nil {
+		// For any error to read fsMeta, set default ETag and proceed.
+		fsMeta = fs.defaultFsJSON(object)
+	}
+
+	// clean fsMeta.Meta of tag key, before updating the new tags
+	delete(fsMeta.Meta, xhttp.AmzObjectTagging)
+
+	// Do not update for empty tags
+	if tags != "" {
+		fsMeta.Meta[xhttp.AmzObjectTagging] = tags
+	}
+
+	if _, err = fsMeta.WriteTo(wlk); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+	return nil
+}
+
+// DeleteObjectTag - delete object tags from an existing object
+func (fs *FSObjects) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	return fs.PutObjectTag(ctx, bucket, object, "")
+}
+
 // ReloadFormat - no-op for fs, Valid only for XL.
 func (fs *FSObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
 	logger.LogIf(ctx, NotImplemented{})

cmd/gateway-common.go

@@ -304,9 +304,12 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
 	if err != nil {
 		return false
 	}
-	if _, err = clnt.Do(req); err != nil {
+	resp, err := clnt.Do(req)
+	if err != nil {
+		clnt.CloseIdleConnections()
 		return !xnet.IsNetworkOrHostDown(err)
 	}
+	xhttp.DrainBody(resp.Body)
 	return true
 }
 

cmd/gateway-unsupported.go

@@ -24,6 +24,7 @@ import (
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 // GatewayLocker implements custom NeNSLock implementation
@@ -45,6 +46,12 @@ func NewGatewayLayerWithLocker(gwLayer ObjectLayer) ObjectLayer {
 // GatewayUnsupported list of unsupported call stubs for gateway.
 type GatewayUnsupported struct{}
 
+// CrawlAndGetDataUsage - crawl is not implemented for gateway
+func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+	logger.CriticalIf(ctx, errors.New("not implemented"))
+	return DataUsageInfo{}
+}
+
 // NewNSLock is a dummy stub for gateway.
 func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, object string) RWLocker {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
@@ -173,6 +180,24 @@ func (a GatewayUnsupported) GetMetrics(ctx context.Context) (*Metrics, error) {
 	return &Metrics{}, NotImplemented{}
 }
 
+// PutObjectTag - not implemented.
+func (a GatewayUnsupported) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	logger.LogIf(ctx, NotImplemented{})
+	return NotImplemented{}
+}
+
+// GetObjectTag - not implemented.
+func (a GatewayUnsupported) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	logger.LogIf(ctx, NotImplemented{})
+	return tagging.Tagging{}, NotImplemented{}
+}
+
+// DeleteObjectTag - not implemented.
+func (a GatewayUnsupported) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	logger.LogIf(ctx, NotImplemented{})
+	return NotImplemented{}
+}
+
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (a GatewayUnsupported) IsNotificationSupported() bool {
 	return false

cmd/gateway/b2/gateway-b2.go

@@ -127,6 +127,7 @@ type b2Objects struct {
 	b2Client   *b2.B2
 	httpClient *http.Client
 	ctx        context.Context
+	buckets    []*b2.Bucket
 }
 
 // Convert B2 errors to minio object layer errors.
@@ -257,11 +258,14 @@ func (l *b2Objects) listBuckets(ctx context.Context, err error) ([]*b2.Bucket, e
 			return nil, rerr
 		}
 	}
-	bktList, lerr := l.b2Client.ListBuckets(l.ctx)
-	if lerr != nil {
-		return l.listBuckets(ctx, lerr)
+	if len(l.buckets) == 0 {
+		bktList, lerr := l.b2Client.ListBuckets(l.ctx)
+		if lerr != nil {
+			return l.listBuckets(ctx, lerr)
+		}
+		l.buckets = bktList
 	}
-	return bktList, nil
+	return l.buckets, nil
 }
 
 // Bucket - is a helper which provides a *Bucket instance
@@ -316,6 +320,20 @@ func (l *b2Objects) DeleteBucket(ctx context.Context, bucket string) error {
 		return err
 	}
 	err = bkt.DeleteBucket(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		err = bkt.DeleteBucket(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket)
+	}
+
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err, bucket)
 }
@@ -328,8 +346,18 @@ func (l *b2Objects) ListObjects(ctx context.Context, bucket string, prefix strin
 	}
 	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
-		logger.LogIf(ctx, lerr)
-		return loi, b2ToObjectError(lerr, bucket)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return loi, err
+		}
+
+		files, next, lerr = bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+		if lerr != nil {
+			logger.LogIf(ctx, lerr)
+			return loi, b2ToObjectError(lerr, bucket)
+		}
 	}
 	loi.IsTruncated = next != ""
 	loi.NextMarker = next
@@ -370,8 +398,18 @@ func (l *b2Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	}
 	files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
 	if lerr != nil {
-		logger.LogIf(ctx, lerr)
-		return loi, b2ToObjectError(lerr, bucket)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return loi, err
+		}
+
+		files, next, lerr = bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
+		if lerr != nil {
+			logger.LogIf(ctx, lerr)
+			return loi, b2ToObjectError(lerr, bucket)
+		}
 	}
 	loi.IsTruncated = next != ""
 	loi.ContinuationToken = continuationToken
@@ -433,8 +471,18 @@ func (l *b2Objects) GetObject(ctx context.Context, bucket string, object string,
 	}
 	reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		reader, err = bkt.DownloadFileByName(l.ctx, object, startOffset, length)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return b2ToObjectError(err, bucket, object)
+		}
 	}
 	defer reader.Close()
 	_, err = io.Copy(writer, reader)
@@ -451,8 +499,18 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 
 	f, _, err := bkt.ListFileNames(l.ctx, 1, object, "", "")
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return objInfo, err
+		}
+
+		f, _, err = bkt.ListFileNames(l.ctx, 1, object, "", "")
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, b2ToObjectError(err, bucket, object)
+		}
 	}
 
 	// B2's list will return the next item in the bucket if the object doesn't
@@ -466,8 +524,18 @@ func (l *b2Objects) GetObjectInfo(ctx context.Context, bucket string, object str
 
 	fi, err := bkt.File(f[0].ID, object).GetFileInfo(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return objInfo, err
+		}
+
+		fi, err = bkt.File(f[0].ID, object).GetFileInfo(l.ctx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, b2ToObjectError(err, bucket, object)
+		}
 	}
 	return minio.ObjectInfo{
 		Bucket:      bucket,
@@ -557,8 +625,18 @@ func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string,
 	var u *b2.URL
 	u, err = bkt.GetUploadURL(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return objInfo, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return objInfo, err
+		}
+
+		u, err = bkt.GetUploadURL(l.ctx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return objInfo, b2ToObjectError(err, bucket, object)
+		}
 	}
 
 	hr := newB2Reader(data, data.Size())
@@ -597,6 +675,19 @@ func (l *b2Objects) DeleteObject(ctx context.Context, bucket string, object stri
 	// If we hide the file we'll conform to B2's versioning policy, it also
 	// saves an additional call to check if the file exists first
 	_, err = bkt.HideFile(l.ctx, object)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		_, err = bkt.HideFile(l.ctx, object)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err, bucket, object)
 }
@@ -625,8 +716,18 @@ func (l *b2Objects) ListMultipartUploads(ctx context.Context, bucket string, pre
 	}
 	largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return lmi, b2ToObjectError(err, bucket)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return lmi, err
+		}
+
+		largeFiles, nextMarker, err = bkt.ListUnfinishedLargeFiles(l.ctx, maxUploads, uploadIDMarker)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return lmi, b2ToObjectError(err, bucket)
+		}
 	}
 	lmi = minio.ListMultipartsInfo{
 		MaxUploads: maxUploads,
@@ -661,8 +762,18 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
 	delete(opts.UserDefined, "content-type")
 	lf, err := bkt.StartLargeFile(l.ctx, object, contentType, opts.UserDefined)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return uploadID, b2ToObjectError(err, bucket, object)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return uploadID, err
+		}
+
+		lf, err = bkt.StartLargeFile(l.ctx, object, contentType, opts.UserDefined)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return uploadID, b2ToObjectError(err, bucket, object)
+		}
 	}
 
 	return lf.ID, nil
@@ -678,8 +789,18 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
 
 	fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return pi, b2ToObjectError(err, bucket, object, uploadID)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return pi, err
+		}
+
+		fc, err = bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return pi, b2ToObjectError(err, bucket, object, uploadID)
+		}
 	}
 
 	hr := newB2Reader(data, data.Size())
@@ -714,8 +835,18 @@ func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object s
 	partNumberMarker++
 	partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
 	if err != nil {
-		logger.LogIf(ctx, err)
-		return lpi, b2ToObjectError(err, bucket, object, uploadID)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return lpi, err
+		}
+
+		partsList, next, err = bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return lpi, b2ToObjectError(err, bucket, object, uploadID)
+		}
 	}
 	if next != 0 {
 		lpi.IsTruncated = true
@@ -738,6 +869,19 @@ func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
 		return err
 	}
 	err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+
+		err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err, bucket, object, uploadID)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err, bucket, object, uploadID)
 }
@@ -762,8 +906,17 @@ func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
-		logger.LogIf(ctx, err)
-		return oi, b2ToObjectError(err, bucket, object, uploadID)
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return oi, err
+		}
+
+		if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
+			logger.LogIf(ctx, err)
+			return oi, b2ToObjectError(err, bucket, object, uploadID)
+		}
 	}
 
 	return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
@@ -806,6 +959,20 @@ func (l *b2Objects) SetBucketPolicy(ctx context.Context, bucket string, bucketPo
 	}
 	bkt.Type = bucketTypeReadOnly
 	_, err = bkt.Update(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+		bkt.Type = bucketTypeReadOnly
+
+		_, err = bkt.Update(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err)
 }
@@ -854,6 +1021,20 @@ func (l *b2Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
 	}
 	bkt.Type = bucketTypePrivate
 	_, err = bkt.Update(l.ctx)
+	if err != nil {
+		l.buckets = []*b2.Bucket{}
+
+		bkt, err := l.Bucket(ctx, bucket)
+		if err != nil {
+			return err
+		}
+		bkt.Type = bucketTypePrivate
+
+		_, err = bkt.Update(l.ctx)
+
+		logger.LogIf(ctx, err)
+		return b2ToObjectError(err)
+	}
 	logger.LogIf(ctx, err)
 	return b2ToObjectError(err)
 }

cmd/gateway/hdfs/gateway-hdfs.go

@@ -196,7 +196,7 @@ func (g *HDFS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error
 
 // Production - hdfs gateway is production ready.
 func (g *HDFS) Production() bool {
-	return false
+	return true
 }
 
 func (n *hdfsObjects) Shutdown(ctx context.Context) error {
@@ -355,8 +355,23 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 
 // ListObjects lists all blobs in HDFS bucket filtered by prefix.
 func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
+	if _, err := n.clnt.Stat(minio.PathJoin(hdfsSeparator, bucket)); err != nil {
+		return loi, hdfsToObjectErr(ctx, err, bucket)
+	}
+
 	getObjectInfo := func(ctx context.Context, bucket, entry string) (minio.ObjectInfo, error) {
-		return n.GetObjectInfo(ctx, bucket, entry, minio.ObjectOptions{})
+		fi, err := n.clnt.Stat(minio.PathJoin(hdfsSeparator, bucket, entry))
+		if err != nil {
+			return minio.ObjectInfo{}, hdfsToObjectErr(ctx, err, bucket, entry)
+		}
+		return minio.ObjectInfo{
+			Bucket:  bucket,
+			Name:    entry,
+			ModTime: fi.ModTime(),
+			Size:    fi.Size(),
+			IsDir:   fi.IsDir(),
+			AccTime: fi.(*hdfs.FileInfo).AccessTime(),
+		}, nil
 	}
 
 	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), getObjectInfo, getObjectInfo)

cmd/generic-handlers.go

@@ -467,8 +467,8 @@ func ignoreNotImplementedBucketResources(req *http.Request) bool {
 // Checks requests for not implemented Object resources
 func ignoreNotImplementedObjectResources(req *http.Request) bool {
 	for name := range req.URL.Query() {
-		// Enable GetObjectACL and GetObjectTagging dummy calls specifically.
-		if (name == "acl" || name == "tagging") && req.Method == http.MethodGet {
+		// Enable GetObjectACL dummy call specifically.
+		if name == "acl" && req.Method == http.MethodGet {
 			return false
 		}
 		if notimplementedObjectResourceNames[name] {
@@ -497,7 +497,6 @@ var notimplementedBucketResourceNames = map[string]bool{
 var notimplementedObjectResourceNames = map[string]bool{
 	"acl":     true,
 	"restore": true,
-	"tagging": true,
 	"torrent": true,
 }
 
@@ -632,11 +631,11 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 		switch r.Method {
 		case http.MethodPut:
 			if getRequestAuthType(r) == authTypeJWT {
-				bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/upload"))
+				bucket, _ = path2BucketObjectWithBasePath(minioReservedBucketPath+"/upload", r.URL.Path)
 			}
 		case http.MethodGet:
 			if t := r.URL.Query().Get("token"); t != "" {
-				bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/download"))
+				bucket, _ = path2BucketObjectWithBasePath(minioReservedBucketPath+"/download", r.URL.Path)
 			} else if getRequestAuthType(r) != authTypeJWT && !strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
 				bucket, _ = request2BucketObjectName(r)
 			}
@@ -688,7 +687,7 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
 	// requests have target bucket and object in URI and source details are in
 	// header fields
 	if r.Method == http.MethodPut && r.Header.Get(xhttp.AmzCopySource) != "" {
-		bucket, object = urlPath2BucketObjectName(r.Header.Get(xhttp.AmzCopySource))
+		bucket, object = path2BucketObject(r.Header.Get(xhttp.AmzCopySource))
 		if bucket == "" || object == "" {
 			f.handler.ServeHTTP(w, r)
 			return

cmd/globals.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/event"
+	"github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/pubsub"
 )
 
@@ -201,7 +202,7 @@ var (
 	// Is worm enabled
 	globalWORMEnabled bool
 
-	globalBucketObjectLockConfig = newBucketObjectLockConfig()
+	globalBucketObjectLockConfig = objectlock.NewBucketObjectLockConfig()
 
 	// Disk cache drives
 	globalCacheConfig cache.Config

cmd/handler-utils.go

@@ -34,6 +34,12 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/tagging"
+)
+
+const (
+	copyDirective    = "COPY"
+	replaceDirective = "REPLACE"
 )
 
 // Parses location constraint from the incoming reader.
@@ -69,47 +75,27 @@ var supportedHeaders = []string{
 	"content-encoding",
 	"content-disposition",
 	xhttp.AmzStorageClass,
+	xhttp.AmzObjectTagging,
 	"expires",
 	// Add more supported headers here.
 }
 
-// isMetadataDirectiveValid - check if metadata-directive is valid.
-func isMetadataDirectiveValid(h http.Header) bool {
-	_, ok := h[http.CanonicalHeaderKey(xhttp.AmzMetadataDirective)]
-	if ok {
-		// Check atleast set metadata-directive is valid.
-		return (isMetadataCopy(h) || isMetadataReplace(h))
-	}
-	// By default if x-amz-metadata-directive is not we
+// isDirectiveValid - check if tagging-directive is valid.
+func isDirectiveValid(v string) bool {
+	// Check if set metadata-directive is valid.
+	return isDirectiveCopy(v) || isDirectiveReplace(v)
+}
+
+// Check if the directive COPY is requested.
+func isDirectiveCopy(value string) bool {
+	// By default if directive is not set we
 	// treat it as 'COPY' this function returns true.
-	return true
+	return value == copyDirective || value == ""
 }
 
-// Check if the metadata COPY is requested.
-func isMetadataCopy(h http.Header) bool {
-	return h.Get(xhttp.AmzMetadataDirective) == "COPY"
-}
-
-// Check if the metadata REPLACE is requested.
-func isMetadataReplace(h http.Header) bool {
-	return h.Get(xhttp.AmzMetadataDirective) == "REPLACE"
-}
-
-// Splits an incoming path into bucket and object components.
-func path2BucketAndObject(path string) (bucket, object string) {
-	// Skip the first element if it is '/', split the rest.
-	path = strings.TrimPrefix(path, SlashSeparator)
-	pathComponents := strings.SplitN(path, SlashSeparator, 2)
-
-	// Save the bucket and object extracted from path.
-	switch len(pathComponents) {
-	case 1:
-		bucket = pathComponents[0]
-	case 2:
-		bucket = pathComponents[0]
-		object = pathComponents[1]
-	}
-	return bucket, object
+// Check if the directive REPLACE is requested.
+func isDirectiveReplace(value string) bool {
+	return value == replaceDirective
 }
 
 // userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys.
@@ -174,6 +160,26 @@ func extractMetadataFromMap(ctx context.Context, v map[string][]string, m map[st
 	return nil
 }
 
+// extractTags extracts tag key and value from given http header. It then
+// - Parses the input format X-Amz-Tagging:"Key1=Value1&Key2=Value2" into a map[string]string
+// with entries in the format X-Amg-Tag-Key1:Value1, X-Amz-Tag-Key2:Value2
+// - Validates the tags
+// - Returns the Tag in original string format "Key1=Value1&Key2=Value2"
+func extractTags(ctx context.Context, tags string) (string, error) {
+	// Check if the metadata has tagging related header
+	if tags != "" {
+		tagging, err := tagging.FromString(tags)
+		if err != nil {
+			return "", err
+		}
+		if err := tagging.Validate(); err != nil {
+			return "", err
+		}
+		return tagging.String(), nil
+	}
+	return "", nil
+}
+
 // The Query string for the redirect URL the client is
 // redirected on successful upload.
 func getRedirectPostRawQuery(objInfo ObjectInfo) string {

cmd/http/headers.go

@@ -56,6 +56,11 @@ const (
 	// S3 storage class
 	AmzStorageClass = "x-amz-storage-class"
 
+	// S3 object tagging
+	AmzObjectTagging = "X-Amz-Tagging"
+	AmzTagCount      = "X-Amz-Tag-Count"
+	AmzTagDirective  = "X-Amz-Tagging-Directive"
+
 	// S3 extensions
 	AmzCopySourceIfModifiedSince   = "x-amz-copy-source-if-modified-since"
 	AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since"

cmd/http/server.go

@@ -87,15 +87,19 @@ func (srv *Server) Start() (err error) {
 	// Wrap given handler to do additional
 	// * return 503 (service unavailable) if the server in shutdown.
 	wrappedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		atomic.AddInt32(&srv.requestCount, 1)
-		defer atomic.AddInt32(&srv.requestCount, -1)
-
-		// If server is in shutdown, return 503 (service unavailable)
+		// If server is in shutdown.
 		if atomic.LoadUint32(&srv.inShutdown) != 0 {
-			w.WriteHeader(http.StatusServiceUnavailable)
+			// To indicate disable keep-alives
+			w.Header().Set("Connection", "close")
+			w.WriteHeader(http.StatusForbidden)
+			w.Write([]byte(http.ErrServerClosed.Error()))
+			w.(http.Flusher).Flush()
 			return
 		}
 
+		atomic.AddInt32(&srv.requestCount, 1)
+		defer atomic.AddInt32(&srv.requestCount, -1)
+
 		// Handle request using passed handler.
 		handler.ServeHTTP(w, r)
 	})
@@ -117,13 +121,13 @@ func (srv *Server) Shutdown() error {
 	srv.listenerMutex.Lock()
 	if srv.listener == nil {
 		srv.listenerMutex.Unlock()
-		return errors.New("server not initialized")
+		return http.ErrServerClosed
 	}
 	srv.listenerMutex.Unlock()
 
 	if atomic.AddUint32(&srv.inShutdown, 1) > 1 {
 		// shutdown in progress
-		return errors.New("http server already in shutdown")
+		return http.ErrServerClosed
 	}
 
 	// Close underneath HTTP listener.

cmd/iam.go

@@ -1367,8 +1367,13 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 		return true
 	}
 
-	// With claims set, we should do STS related checks and validation.
-	if _, ok := args.Claims["aud"]; ok {
+	// If the credential is temporary, perform STS related checks.
+	ok, err := sys.IsTempUser(args.AccountName)
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return false
+	}
+	if ok {
 		return sys.IsAllowedSTS(args)
 	}
 
@@ -1449,9 +1454,6 @@ func (sys *IAMSys) updateGroupMembershipsMap(group string, gi *GroupInfo) {
 // removeGroupFromMembershipsMap - removes the group from every member
 // in the cache. IMPORTANT: Assumes sys.Lock() is held by caller.
 func (sys *IAMSys) removeGroupFromMembershipsMap(group string) {
-	if _, ok := sys.iamUserGroupMemberships[group]; !ok {
-		return
-	}
 	for member, groups := range sys.iamUserGroupMemberships {
 		if !groups.Contains(group) {
 			continue

cmd/logger/target/http/http.go

@@ -66,6 +66,7 @@ func (h *Target) startHTTPLogger() {
 
 			resp, err := h.client.Do(req)
 			if err != nil {
+				h.client.CloseIdleConnections()
 				continue
 			}
 

cmd/notification.go

@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -668,7 +669,7 @@ func (sys *NotificationSys) initBucketObjectLockConfig(objAPI ObjectLayer) error
 
 		if string(bucketObjLockData) != bucketObjectLockEnabledConfig {
 			// this should never happen
-			logger.LogIf(ctx, errMalformedBucketObjectConfig)
+			logger.LogIf(ctx, objectlock.ErrMalformedBucketObjectConfig)
 			continue
 		}
 
@@ -677,17 +678,17 @@ func (sys *NotificationSys) initBucketObjectLockConfig(objAPI ObjectLayer) error
 
 		if err != nil {
 			if err == errConfigNotFound {
-				globalBucketObjectLockConfig.Set(bucket.Name, Retention{})
+				globalBucketObjectLockConfig.Set(bucket.Name, objectlock.Retention{})
 				continue
 			}
 			return err
 		}
 
-		config, err := parseObjectLockConfig(bytes.NewReader(configData))
+		config, err := objectlock.ParseObjectLockConfig(bytes.NewReader(configData))
 		if err != nil {
 			return err
 		}
-		retention := Retention{}
+		retention := objectlock.Retention{}
 		if config.Rule != nil {
 			retention = config.ToRetention()
 		}
@@ -874,7 +875,7 @@ func (sys *NotificationSys) Send(args eventArgs) []event.TargetIDErr {
 }
 
 // PutBucketObjectLockConfig - put bucket object lock configuration to all peers.
-func (sys *NotificationSys) PutBucketObjectLockConfig(ctx context.Context, bucketName string, retention Retention) {
+func (sys *NotificationSys) PutBucketObjectLockConfig(ctx context.Context, bucketName string, retention objectlock.Retention) {
 	g := errgroup.WithNErrs(len(sys.peerClients))
 	for index, client := range sys.peerClients {
 		if client == nil {

cmd/object-api-datatypes.go

@@ -154,6 +154,9 @@ type ObjectInfo struct {
 	// User-Defined metadata
 	UserDefined map[string]string
 
+	// User-Defined object tags
+	UserTags string
+
 	// List of individual parts, maximum size of upto 10,000
 	Parts []ObjectPartInfo `json:"-"`
 

cmd/object-api-interface.go

@@ -25,6 +25,7 @@ import (
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 // CheckCopyPreconditionFn returns true if copy precondition check failed.
@@ -56,6 +57,7 @@ type ObjectLayer interface {
 
 	// Storage operations.
 	Shutdown(context.Context) error
+	CrawlAndGetDataUsage(context.Context, <-chan struct{}) DataUsageInfo
 	StorageInfo(context.Context) StorageInfo
 
 	// Bucket operations.
@@ -125,4 +127,9 @@ type ObjectLayer interface {
 
 	// Check Readiness
 	IsReady(ctx context.Context) bool
+
+	// ObjectTagging operations
+	PutObjectTag(context.Context, string, string, string) error
+	GetObjectTag(context.Context, string, string) (tagging.Tagging, error)
+	DeleteObjectTag(context.Context, string, string) error
 }

cmd/object-api-utils.go

@@ -239,8 +239,8 @@ func getCompleteMultipartMD5(parts []CompletePart) string {
 func cleanMetadata(metadata map[string]string) map[string]string {
 	// Remove STANDARD StorageClass
 	metadata = removeStandardStorageClass(metadata)
-	// Clean meta etag keys 'md5Sum', 'etag', "expires".
-	return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires")
+	// Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging".
+	return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging)
 }
 
 // Filter X-Amz-Storage-Class field only if it is set to STANDARD.

cmd/object-handlers.go

@@ -44,10 +44,13 @@ import (
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
+	"github.com/minio/minio/pkg/objectlock"
+
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/ioutil"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/s3select"
+	"github.com/minio/minio/pkg/tagging"
 	sha256 "github.com/minio/sha256-simd"
 	"github.com/minio/sio"
 )
@@ -209,8 +212,10 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		return
 	}
 	getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
+	legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
+
 	// filter object lock metadata if permission does not permit
-	objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
+	objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
 
 	if err = s3Select.Open(getObject); err != nil {
 		if serr, ok := err.(s3select.SelectError); ok {
@@ -353,7 +358,10 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 
 	// filter object lock metadata if permission does not permit
 	getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
-	objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
+	legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
+
+	// filter object lock metadata if permission does not permit
+	objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
 
 	if objectAPI.IsEncryptionSupported() {
 		objInfo.UserDefined = CleanMinioInternalMetadataKeys(objInfo.UserDefined)
@@ -518,7 +526,10 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// filter object lock metadata if permission does not permit
 	getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
-	objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
+	legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
+
+	// filter object lock metadata if permission does not permit
+	objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
 
 	if objectAPI.IsEncryptionSupported() {
 		if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
@@ -594,13 +605,13 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 
 	// if x-amz-metadata-directive says REPLACE then
 	// we extract metadata from the input headers.
-	if isMetadataReplace(r.Header) {
+	if isDirectiveReplace(r.Header.Get(xhttp.AmzMetadataDirective)) {
 		return extractMetadata(ctx, r)
 	}
 
 	// if x-amz-metadata-directive says COPY then we
 	// return the default metadata.
-	if isMetadataCopy(r.Header) {
+	if isDirectiveCopy(r.Header.Get(xhttp.AmzMetadataDirective)) {
 		return defaultMeta, nil
 	}
 
@@ -608,6 +619,24 @@ func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta m
 	return defaultMeta, nil
 }
 
+// Extract tags relevant for an CopyObject operation based on conditional
+// header values specified in X-Amz-Tagging-Directive.
+func getCpObjTagsFromHeader(ctx context.Context, r *http.Request, tags string) (string, error) {
+	// if x-amz-tagging-directive says REPLACE then
+	// we extract tags from the input headers.
+	if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) {
+		if tags := r.Header.Get(xhttp.AmzObjectTagging); tags != "" {
+			return extractTags(ctx, tags)
+		}
+		// Copy is default behavior if x-amz-tagging-directive is set, but x-amz-tagging is
+		// is not set
+		return tags, nil
+	}
+
+	// Copy is default behavior if x-amz-tagging-directive is not set.
+	return tags, nil
+}
+
 // Returns a minio-go Client configured to access remote host described by destDNSRecord
 // Applicable only in a federated deployment
 var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) {
@@ -714,7 +743,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 	}
 
-	srcBucket, srcObject := path2BucketAndObject(cpSrcPath)
+	srcBucket, srcObject := path2BucketObject(cpSrcPath)
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
@@ -727,11 +756,17 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	// Check if metadata directive is valid.
-	if !isMetadataDirectiveValid(r.Header) {
+	if !isDirectiveValid(r.Header.Get(xhttp.AmzMetadataDirective)) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMetadataDirective), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
+	// check if tag directive is valid
+	if !isDirectiveValid(r.Header.Get(xhttp.AmzTagDirective)) {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidTagDirective), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	// This request header needs to be set prior to setting ObjectOptions
 	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
@@ -785,7 +820,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	defer gr.Close()
 	srcInfo := gr.ObjInfo
 
-	/// maximum Upload size for object in a single CopyObject operation.
+	// maximum Upload size for object in a single CopyObject operation.
 	if isMaxObjectSize(srcInfo.Size) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL, guessIsBrowserReq(r))
 		return
@@ -958,20 +993,34 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
+	tags, err := getCpObjTagsFromHeader(ctx, r, srcInfo.UserTags)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if tags != "" {
+		srcInfo.UserDefined[xhttp.AmzObjectTagging] = tags
+	}
+
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
-	isCpy := true
-	getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, srcBucket, srcObject)
-	srcInfo.UserDefined = filterObjectLockMetadata(ctx, r, srcBucket, srcObject, srcInfo.UserDefined, isCpy, getRetPerms)
+	srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
 	retPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectRetentionAction)
+	holdPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectLegalHoldAction)
+
 	// apply default bucket configuration/governance headers for dest side.
-	retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms)
+	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms)
 	if s3Err == ErrNone && retentionMode != "" {
 		srcInfo.UserDefined[xhttp.AmzObjectLockMode] = string(retentionMode)
 		srcInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate] = retentionDate.UTC().Format(time.RFC3339)
 	}
+	if s3Err == ErrNone && legalHold.Status != "" {
+		srcInfo.UserDefined[xhttp.AmzObjectLockLegalHold] = string(legalHold.Status)
+	}
 	if s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
@@ -989,12 +1038,13 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Ensure that metadata does not contain sensitive information
 	crypto.RemoveSensitiveEntries(srcInfo.UserDefined)
-	// Check if x-amz-metadata-directive was not set to REPLACE and source,
-	// desination are same objects. Apply this restriction also when
+	// Check if x-amz-metadata-directive or x-amz-tagging-directive was not set to REPLACE and source,
+	// destination are same objects. Apply this restriction also when
 	// metadataOnly is true indicating that we are not overwriting the object.
 	// if encryption is enabled we do not need explicit "REPLACE" metadata to
 	// be enabled as well - this is to allow for key-rotation.
-	if !isMetadataReplace(r.Header) && srcInfo.metadataOnly && !crypto.IsEncrypted(srcInfo.UserDefined) {
+	if !isDirectiveReplace(r.Header.Get(xhttp.AmzMetadataDirective)) && !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) &&
+		srcInfo.metadataOnly && !crypto.IsEncrypted(srcInfo.UserDefined) {
 		// If x-amz-metadata-directive is not set to REPLACE then we need
 		// to error out if source and destination are same.
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyDest), r.URL, guessIsBrowserReq(r))
@@ -1142,6 +1192,14 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	if tags := r.Header.Get(http.CanonicalHeaderKey(xhttp.AmzObjectTagging)); tags != "" {
+		metadata[xhttp.AmzObjectTagging], err = extractTags(ctx, tags)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
+
 	if rAuthType == authTypeStreamingSigned {
 		if contentEncoding, ok := metadata["content-encoding"]; ok {
 			contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
@@ -1251,11 +1309,16 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		putObject = api.CacheAPI().PutObject
 	}
 	retPerms := isPutActionAllowed(rAuthType, bucket, object, r, iampolicy.PutObjectRetentionAction)
-	retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, getObjectInfo, retPerms)
+	holdPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
+
+	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
 	if s3Err == ErrNone && retentionMode != "" {
 		metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
 		metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(time.RFC3339)
 	}
+	if s3Err == ErrNone && legalHold.Status != "" {
+		metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
+	}
 	if s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
@@ -1409,11 +1472,16 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		return
 	}
 	retPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectRetentionAction)
-	retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms)
+	holdPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
+
+	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms, holdPerms)
 	if s3Err == ErrNone && retentionMode != "" {
 		metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
 		metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(time.RFC3339)
 	}
+	if s3Err == ErrNone && legalHold.Status != "" {
+		metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
+	}
 	if s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
@@ -1510,7 +1578,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		}
 	}
 
-	srcBucket, srcObject := path2BucketAndObject(cpSrcPath)
+	srcBucket, srcObject := path2BucketObject(cpSrcPath)
 	// If source object is empty or bucket is empty, reply back invalid copy source.
 	if srcObject == "" || srcBucket == "" {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL, guessIsBrowserReq(r))
@@ -2215,14 +2283,16 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	// Reject retention or governance headers if set, CompleteMultipartUpload spec
-	// does not use these headers, and should not be passed down to checkPutObjectRetentionAllowed
-	if isObjectLockRequested(r.Header) || isObjectLockGovernanceBypassSet(r.Header) {
+	// does not use these headers, and should not be passed down to checkPutObjectLockAllowed
+	if objectlock.IsObjectLockRequested(r.Header) || objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL, guessIsBrowserReq(r))
 		return
 	}
 	// Enforce object lock governance in case a competing upload finalized first.
 	retPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectRetentionAction)
-	if _, _, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms); s3Err != ErrNone {
+	holdPerms := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.PutObjectLegalHoldAction)
+
+	if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, retPerms, holdPerms); s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
 	}
@@ -2467,23 +2537,81 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Check permissions to perform this legal hold operation
+	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction); s3Err != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Get Content-Md5 sent by client and verify if valid
+	md5Bytes, err := checkValidMD5(r.Header)
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	if _, ok := globalBucketObjectLockConfig.Get(bucket); !ok {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	legalHold, err := objectlock.ParseObjectLegalHold(r.Body)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// verify Content-MD5 sum of request body if this header set
+	if len(md5Bytes) > 0 {
+		data, err := xml.Marshal(legalHold)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		if hex.EncodeToString(md5Bytes) != getMD5Hash(data) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
-
 	opts, err := getOpts(ctx, r, bucket, object)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	if _, err = getObjectInfo(ctx, bucket, object, opts); err != nil {
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+	objInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
+	objInfo.metadataOnly = true
+
+	if _, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{}, ObjectOptions{}); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+	writeSuccessNoContent(w)
+	// Notify object  event.
+	sendEvent(eventArgs{
+		EventName:    event.ObjectCreatedPutLegalHold,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
+
 }
 
 // GetObjectLegalHoldHandler - get legal hold configuration to object,
@@ -2506,6 +2634,10 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
 		return
 	}
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
 
 	getObjectInfo := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
@@ -2518,12 +2650,25 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, err = getObjectInfo(ctx, bucket, object, opts); err != nil {
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
 
-	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
+	legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
+
+	writeSuccessResponseXML(w, encodeResponse(legalHold))
+	// Notify object legal hold accessed via a GET request.
+	sendEvent(eventArgs{
+		EventName:    event.ObjectAccessedGetLegalHold,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
 }
 
 // PutObjectRetentionHandler - set object hold configuration to object,
@@ -2568,7 +2713,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	objRetention, err := parseObjectRetention(r.Body)
+	objRetention, err := objectlock.ParseObjectRetention(r.Body)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		apiErr.Description = err.Error()
@@ -2660,7 +2805,7 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	retention := getObjectRetentionMeta(objInfo.UserDefined)
+	retention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
 
 	writeSuccessResponseXML(w, encodeResponse(retention))
 	// Notify object retention accessed via a GET request.
@@ -2674,3 +2819,103 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
 		Host:         handlers.GetSourceIP(r),
 	})
 }
+
+// GetObjectTaggingHandler - GET object tagging
+func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetObjectTagging")
+	defer logger.AuditLog(w, r, "GetObjectTagging", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow getObjectTagging if policy action is set.
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectTaggingAction, bucket, object); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Get object tags
+	tags, err := objAPI.GetObjectTag(ctx, bucket, object)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	writeSuccessResponseXML(w, encodeResponse(tags))
+}
+
+// PutObjectTaggingHandler - PUT object tagging
+func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutObjectTagging")
+	defer logger.AuditLog(w, r, "PutObjectTagging", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow putObjectTagging if policy action is set
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectTaggingAction, bucket, object); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	tagging, err := tagging.ParseTagging(io.LimitReader(r.Body, r.ContentLength))
+
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Put object tags
+	err = objAPI.PutObjectTag(ctx, bucket, object, tagging.String())
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}
+
+// DeleteObjectTaggingHandler - DELETE object tagging
+func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteObjectTagging")
+	defer logger.AuditLog(w, r, "DeleteObjectTagging", mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	object := vars["object"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Allow deleteObjectTagging if policy action is set
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteObjectTaggingAction, bucket, object); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	// Delete object tags
+	err := objAPI.DeleteObjectTag(ctx, bucket, object)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
+	writeSuccessResponseHeadersOnly(w)
+}

cmd/object-lock.go

@@ -18,387 +18,12 @@ package cmd
 
 import (
 	"context"
-	"encoding/xml"
-	"errors"
-	"fmt"
-	"io"
 	"net/http"
-	"strings"
-	"sync"
-	"time"
 
-	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/env"
+	"github.com/minio/minio/pkg/objectlock"
 )
 
-// RetentionMode - object retention mode.
-type RetentionMode string
-
-const (
-	// Governance - governance mode.
-	Governance RetentionMode = "GOVERNANCE"
-
-	// Compliance - compliance mode.
-	Compliance RetentionMode = "COMPLIANCE"
-
-	// Invalid - invalid retention mode.
-	Invalid RetentionMode = ""
-)
-
-func parseRetentionMode(modeStr string) (mode RetentionMode) {
-	switch strings.ToUpper(modeStr) {
-	case "GOVERNANCE":
-		mode = Governance
-	case "COMPLIANCE":
-		mode = Compliance
-	default:
-		mode = Invalid
-	}
-	return mode
-}
-
-var (
-	errMalformedBucketObjectConfig = errors.New("Invalid bucket object lock config")
-	errInvalidRetentionDate        = errors.New("Date must be provided in ISO 8601 format")
-	errPastObjectLockRetainDate    = errors.New("the retain until date must be in the future")
-	errUnknownWORMModeDirective    = errors.New("unknown WORM mode directive")
-	errObjectLockMissingContentMD5 = errors.New("Content-MD5 HTTP header is required for Put Object requests with Object Lock parameters")
-	errObjectLockInvalidHeaders    = errors.New("x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied")
-)
-
-const (
-	ntpServerEnv = "MINIO_NTP_SERVER"
-)
-
-var (
-	ntpServer = env.Get(ntpServerEnv, "")
-)
-
-// Retention - bucket level retention configuration.
-type Retention struct {
-	Mode     RetentionMode
-	Validity time.Duration
-}
-
-// IsEmpty - returns whether retention is empty or not.
-func (r Retention) IsEmpty() bool {
-	return r.Mode == "" || r.Validity == 0
-}
-
-// Retain - check whether given date is retainable by validity time.
-func (r Retention) Retain(created time.Time) bool {
-	t, err := UTCNowNTP()
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		// Retain
-		return true
-	}
-	return globalWORMEnabled || created.Add(r.Validity).After(t)
-}
-
-// BucketObjectLockConfig - map of bucket and retention configuration.
-type BucketObjectLockConfig struct {
-	sync.RWMutex
-	retentionMap map[string]Retention
-}
-
-// Set - set retention configuration.
-func (config *BucketObjectLockConfig) Set(bucketName string, retention Retention) {
-	config.Lock()
-	config.retentionMap[bucketName] = retention
-	config.Unlock()
-}
-
-// Get - Get retention configuration.
-func (config *BucketObjectLockConfig) Get(bucketName string) (r Retention, ok bool) {
-	config.RLock()
-	defer config.RUnlock()
-	r, ok = config.retentionMap[bucketName]
-	return r, ok
-}
-
-// Remove - removes retention configuration.
-func (config *BucketObjectLockConfig) Remove(bucketName string) {
-	config.Lock()
-	delete(config.retentionMap, bucketName)
-	config.Unlock()
-}
-
-func newBucketObjectLockConfig() *BucketObjectLockConfig {
-	return &BucketObjectLockConfig{
-		retentionMap: map[string]Retention{},
-	}
-}
-
-// DefaultRetention - default retention configuration.
-type DefaultRetention struct {
-	XMLName xml.Name      `xml:"DefaultRetention"`
-	Mode    RetentionMode `xml:"Mode"`
-	Days    *uint64       `xml:"Days"`
-	Years   *uint64       `xml:"Years"`
-}
-
-// Maximum support retention days and years supported by AWS S3.
-const (
-	// This tested by using `mc lock` command
-	maximumRetentionDays  = 36500
-	maximumRetentionYears = 100
-)
-
-// UnmarshalXML - decodes XML data.
-func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
-	// Make subtype to avoid recursive UnmarshalXML().
-	type defaultRetention DefaultRetention
-	retention := defaultRetention{}
-
-	if err := d.DecodeElement(&retention, &start); err != nil {
-		return err
-	}
-
-	switch string(retention.Mode) {
-	case "GOVERNANCE", "COMPLIANCE":
-	default:
-		return fmt.Errorf("unknown retention mode %v", retention.Mode)
-	}
-
-	if retention.Days == nil && retention.Years == nil {
-		return fmt.Errorf("either Days or Years must be specified")
-	}
-
-	if retention.Days != nil && retention.Years != nil {
-		return fmt.Errorf("either Days or Years must be specified, not both")
-	}
-
-	if retention.Days != nil {
-		if *retention.Days == 0 {
-			return fmt.Errorf("Default retention period must be a positive integer value for 'Days'")
-		}
-		if *retention.Days > maximumRetentionDays {
-			return fmt.Errorf("Default retention period too large for 'Days' %d", *retention.Days)
-		}
-	} else if *retention.Years == 0 {
-		return fmt.Errorf("Default retention period must be a positive integer value for 'Years'")
-	} else if *retention.Years > maximumRetentionYears {
-		return fmt.Errorf("Default retention period too large for 'Years' %d", *retention.Years)
-	}
-
-	*dr = DefaultRetention(retention)
-
-	return nil
-}
-
-// ObjectLockConfig - object lock configuration specified in
-// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html
-type ObjectLockConfig struct {
-	XMLNS             string   `xml:"xmlns,attr,omitempty"`
-	XMLName           xml.Name `xml:"ObjectLockConfiguration"`
-	ObjectLockEnabled string   `xml:"ObjectLockEnabled"`
-	Rule              *struct {
-		DefaultRetention DefaultRetention `xml:"DefaultRetention"`
-	} `xml:"Rule,omitempty"`
-}
-
-// UnmarshalXML - decodes XML data.
-func (config *ObjectLockConfig) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
-	// Make subtype to avoid recursive UnmarshalXML().
-	type objectLockConfig ObjectLockConfig
-	parsedConfig := objectLockConfig{}
-
-	if err := d.DecodeElement(&parsedConfig, &start); err != nil {
-		return err
-	}
-
-	if parsedConfig.ObjectLockEnabled != "Enabled" {
-		return fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element")
-	}
-
-	*config = ObjectLockConfig(parsedConfig)
-	return nil
-}
-
-// ToRetention - convert to Retention type.
-func (config *ObjectLockConfig) ToRetention() (r Retention) {
-	if config.Rule != nil {
-		r.Mode = config.Rule.DefaultRetention.Mode
-
-		t, err := UTCNowNTP()
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			// Do not change any configuration
-			// upon NTP failure.
-			return r
-		}
-
-		if config.Rule.DefaultRetention.Days != nil {
-			r.Validity = t.AddDate(0, 0, int(*config.Rule.DefaultRetention.Days)).Sub(t)
-		} else {
-			r.Validity = t.AddDate(int(*config.Rule.DefaultRetention.Years), 0, 0).Sub(t)
-		}
-	}
-
-	return r
-}
-
-func parseObjectLockConfig(reader io.Reader) (*ObjectLockConfig, error) {
-	config := ObjectLockConfig{}
-	if err := xml.NewDecoder(reader).Decode(&config); err != nil {
-		return nil, err
-	}
-
-	return &config, nil
-}
-
-func newObjectLockConfig() *ObjectLockConfig {
-	return &ObjectLockConfig{
-		ObjectLockEnabled: "Enabled",
-	}
-}
-
-// RetentionDate is a embedded type containing time.Time to unmarshal
-// Date in Retention
-type RetentionDate struct {
-	time.Time
-}
-
-// UnmarshalXML parses date from Expiration and validates date format
-func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
-	var dateStr string
-	err := d.DecodeElement(&dateStr, &startElement)
-	if err != nil {
-		return err
-	}
-	// While AWS documentation mentions that the date specified
-	// must be present in ISO 8601 format, in reality they allow
-	// users to provide RFC 3339 compliant dates.
-	retDate, err := time.Parse(time.RFC3339, dateStr)
-	if err != nil {
-		return errInvalidRetentionDate
-	}
-
-	*rDate = RetentionDate{retDate}
-	return nil
-}
-
-// MarshalXML encodes expiration date if it is non-zero and encodes
-// empty string otherwise
-func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
-	if *rDate == (RetentionDate{time.Time{}}) {
-		return nil
-	}
-	return e.EncodeElement(rDate.Format(time.RFC3339), startElement)
-}
-
-// ObjectRetention specified in
-// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html
-type ObjectRetention struct {
-	XMLNS           string        `xml:"xmlns,attr,omitempty"`
-	XMLName         xml.Name      `xml:"Retention"`
-	Mode            RetentionMode `xml:"Mode,omitempty"`
-	RetainUntilDate RetentionDate `xml:"RetainUntilDate,omitempty"`
-}
-
-func parseObjectRetention(reader io.Reader) (*ObjectRetention, error) {
-	ret := ObjectRetention{}
-	if err := xml.NewDecoder(reader).Decode(&ret); err != nil {
-		return nil, err
-	}
-	if ret.Mode != Compliance && ret.Mode != Governance {
-		return &ret, errUnknownWORMModeDirective
-	}
-
-	t, err := UTCNowNTP()
-	if err != nil {
-		logger.LogIf(context.Background(), err)
-		return &ret, errPastObjectLockRetainDate
-	}
-
-	if ret.RetainUntilDate.Before(t) {
-		return &ret, errPastObjectLockRetainDate
-	}
-
-	return &ret, nil
-}
-
-func isObjectLockRetentionRequested(h http.Header) bool {
-	if _, ok := h[xhttp.AmzObjectLockMode]; ok {
-		return true
-	}
-	if _, ok := h[xhttp.AmzObjectLockRetainUntilDate]; ok {
-		return true
-	}
-	return false
-}
-
-func isObjectLockLegalHoldRequested(h http.Header) bool {
-	_, ok := h[xhttp.AmzObjectLockLegalHold]
-	return ok
-}
-
-func isObjectLockGovernanceBypassSet(h http.Header) bool {
-	v, ok := h[xhttp.AmzObjectLockBypassGovernance]
-	if !ok {
-		return false
-	}
-	val := strings.Join(v, "")
-	return strings.ToLower(val) == "true"
-}
-
-func isObjectLockRequested(h http.Header) bool {
-	return isObjectLockLegalHoldRequested(h) || isObjectLockRetentionRequested(h)
-}
-
-func parseObjectLockRetentionHeaders(h http.Header) (rmode RetentionMode, r RetentionDate, err error) {
-	retMode, ok := h[xhttp.AmzObjectLockMode]
-	if ok {
-		rmode = parseRetentionMode(strings.Join(retMode, ""))
-		if rmode == Invalid {
-			return rmode, r, errUnknownWORMModeDirective
-		}
-	}
-	var retDate time.Time
-	dateStr, ok := h[xhttp.AmzObjectLockRetainUntilDate]
-	if ok {
-		// While AWS documentation mentions that the date specified
-		// must be present in ISO 8601 format, in reality they allow
-		// users to provide RFC 3339 compliant dates.
-		retDate, err = time.Parse(time.RFC3339, strings.Join(dateStr, ""))
-		if err != nil {
-			return rmode, r, errInvalidRetentionDate
-		}
-
-		t, err := UTCNowNTP()
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			return rmode, r, errPastObjectLockRetainDate
-		}
-
-		if retDate.Before(t) {
-			return rmode, r, errPastObjectLockRetainDate
-		}
-	}
-	if len(retMode) == 0 || len(dateStr) == 0 {
-		return rmode, r, errObjectLockInvalidHeaders
-	}
-	return rmode, RetentionDate{retDate}, nil
-
-}
-
-func getObjectRetentionMeta(meta map[string]string) ObjectRetention {
-	var mode RetentionMode
-	var retainTill RetentionDate
-
-	if modeStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
-		mode = parseRetentionMode(modeStr)
-	}
-	if tillStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)]; ok {
-		if t, e := time.Parse(time.RFC3339, tillStr); e == nil {
-			retainTill = RetentionDate{t.UTC()}
-		}
-	}
-	return ObjectRetention{Mode: mode, RetainUntilDate: retainTill}
-}
-
 // enforceRetentionBypassForDelete enforces whether an existing object under governance can be deleted
 // with governance bypass headers set in the request.
 // Objects under site wide WORM can never be overwritten.
@@ -424,16 +49,19 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 		}
 		return oi, toAPIErrorCode(ctx, err)
 	}
-	ret := getObjectRetentionMeta(oi.UserDefined)
-
+	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
+	lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
+	if lhold.Status == objectlock.ON {
+		return oi, ErrObjectLocked
+	}
 	// Here bucket does not support object lock
-	if ret.Mode == Invalid {
+	if ret.Mode == objectlock.Invalid {
 		return oi, ErrNone
 	}
-	if ret.Mode != Compliance && ret.Mode != Governance {
+	if ret.Mode != objectlock.Compliance && ret.Mode != objectlock.Governance {
 		return oi, ErrUnknownWORMModeDirective
 	}
-	t, err := UTCNowNTP()
+	t, err := objectlock.UTCNowNTP()
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return oi, ErrObjectLocked
@@ -441,7 +69,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 	if ret.RetainUntilDate.Before(t) {
 		return oi, ErrNone
 	}
-	if isObjectLockGovernanceBypassSet(r.Header) && ret.Mode == Governance && govBypassPerm == ErrNone {
+	if objectlock.IsObjectLockGovernanceBypassSet(r.Header) && ret.Mode == objectlock.Governance && govBypassPerm == ErrNone {
 		return oi, ErrNone
 	}
 	return oi, ErrObjectLocked
@@ -453,7 +81,7 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted.
-func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerm APIErrorCode, objRetention *ObjectRetention) (oi ObjectInfo, s3Err APIErrorCode) {
+func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, govBypassPerm APIErrorCode, objRetention *objectlock.ObjectRetention) (oi ObjectInfo, s3Err APIErrorCode) {
 	if globalWORMEnabled {
 		return oi, ErrObjectLocked
 	}
@@ -474,24 +102,24 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 		return oi, toAPIErrorCode(ctx, err)
 	}
 
-	ret := getObjectRetentionMeta(oi.UserDefined)
+	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
 	// no retention metadata on object
-	if ret.Mode == Invalid {
+	if ret.Mode == objectlock.Invalid {
 		if _, isWORMBucket := globalBucketObjectLockConfig.Get(bucket); !isWORMBucket {
 			return oi, ErrInvalidBucketObjectLockConfiguration
 		}
 		return oi, ErrNone
 	}
-	t, err := UTCNowNTP()
+	t, err := objectlock.UTCNowNTP()
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return oi, ErrObjectLocked
 	}
 
-	if ret.Mode == Compliance {
+	if ret.Mode == objectlock.Compliance {
 		// Compliance retention mode cannot be changed and retention period cannot be shortened as per
 		// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
-		if objRetention.Mode != Compliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
+		if objRetention.Mode != objectlock.Compliance || objRetention.RetainUntilDate.Before(ret.RetainUntilDate.Time) {
 			return oi, ErrObjectLocked
 		}
 		if objRetention.RetainUntilDate.Before(t) {
@@ -500,8 +128,8 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 		return oi, ErrNone
 	}
 
-	if ret.Mode == Governance {
-		if !isObjectLockGovernanceBypassSet(r.Header) {
+	if ret.Mode == objectlock.Governance {
+		if !objectlock.IsObjectLockGovernanceBypassSet(r.Header) {
 			if objRetention.RetainUntilDate.Before(t) {
 				return oi, ErrInvalidRetentionDate
 			}
@@ -515,111 +143,104 @@ func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, bucket,
 	return oi, ErrNone
 }
 
-// checkPutObjectRetentionAllowed enforces object retention policy for requests with WORM headers
+// checkPutObjectLockAllowed enforces object retention policy and legal hold policy
+// for requests with WORM headers
 // See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-managing.html for the spec.
 // For non-existing objects with object retention headers set, this method returns ErrNone if bucket has
 // locking enabled and user has requisite permissions (s3:PutObjectRetention)
 // If object exists on object store and site wide WORM enabled - this method
 // returns an error. For objects in "Governance" mode, overwrite is allowed if the retention date has expired.
 // For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered.
-func checkPutObjectRetentionAllowed(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr APIErrorCode) (RetentionMode, RetentionDate, APIErrorCode) {
-	var mode RetentionMode
-	var retainDate RetentionDate
+// For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set
+// Both legal hold and retention can be applied independently on an object
+func checkPutObjectLockAllowed(ctx context.Context, r *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.Mode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) {
+	var mode objectlock.Mode
+	var retainDate objectlock.RetentionDate
+	var legalHold objectlock.ObjectLegalHold
 
 	retention, isWORMBucket := globalBucketObjectLockConfig.Get(bucket)
 
-	retentionRequested := isObjectLockRequested(r.Header)
+	retentionRequested := objectlock.IsObjectLockRetentionRequested(r.Header)
+	legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(r.Header)
 
 	var objExists bool
 	opts, err := getOpts(ctx, r, bucket, object)
 	if err != nil {
-		return mode, retainDate, toAPIErrorCode(ctx, err)
+		return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 	}
 	if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil {
 		objExists = true
-		r := getObjectRetentionMeta(objInfo.UserDefined)
-		if globalWORMEnabled || r.Mode == Compliance {
-			return mode, retainDate, ErrObjectLocked
+		r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
+		if globalWORMEnabled || r.Mode == objectlock.Compliance {
+			return mode, retainDate, legalHold, ErrObjectLocked
 		}
 		mode = r.Mode
 		retainDate = r.RetainUntilDate
+		legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined)
+		// Disallow overwriting an object on legal hold
+		if legalHold.Status == "ON" {
+			return mode, retainDate, legalHold, ErrObjectLocked
+		}
+	}
+	if legalHoldRequested {
+		if !isWORMBucket {
+			return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
+		}
+		var lerr error
+		if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(r.Header); lerr != nil {
+			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
+		}
 	}
 	if retentionRequested {
 		if !isWORMBucket {
-			return mode, retainDate, ErrInvalidBucketObjectLockConfiguration
+			return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration
 		}
-		rMode, rDate, err := parseObjectLockRetentionHeaders(r.Header)
+		legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(r.Header)
 		if err != nil {
-			return mode, retainDate, toAPIErrorCode(ctx, err)
+			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
+		}
+		rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(r.Header)
+		if err != nil {
+			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
 		// AWS S3 just creates a new version of object when an object is being overwritten.
-		t, err := UTCNowNTP()
+		t, err := objectlock.UTCNowNTP()
 		if err != nil {
 			logger.LogIf(ctx, err)
-			return mode, retainDate, ErrObjectLocked
+			return mode, retainDate, legalHold, ErrObjectLocked
 		}
 		if objExists && retainDate.After(t) {
-			return mode, retainDate, ErrObjectLocked
+			return mode, retainDate, legalHold, ErrObjectLocked
 		}
-		if rMode == Invalid {
-			return mode, retainDate, toAPIErrorCode(ctx, errObjectLockInvalidHeaders)
+		if rMode == objectlock.Invalid {
+			return mode, retainDate, legalHold, toAPIErrorCode(ctx, objectlock.ErrObjectLockInvalidHeaders)
 		}
 		if retentionPermErr != ErrNone {
-			return mode, retainDate, retentionPermErr
+			return mode, retainDate, legalHold, retentionPermErr
 		}
-		return rMode, rDate, ErrNone
+		return rMode, rDate, legalHold, ErrNone
 	}
 
 	if !retentionRequested && isWORMBucket {
-		if retention.IsEmpty() && (mode == Compliance || mode == Governance) {
-			return mode, retainDate, ErrObjectLocked
+		if retention.IsEmpty() && (mode == objectlock.Compliance || mode == objectlock.Governance) {
+			return mode, retainDate, legalHold, ErrObjectLocked
 		}
 		if retentionPermErr != ErrNone {
-			return mode, retainDate, retentionPermErr
+			return mode, retainDate, legalHold, retentionPermErr
 		}
-		t, err := UTCNowNTP()
+		t, err := objectlock.UTCNowNTP()
 		if err != nil {
 			logger.LogIf(ctx, err)
-			return mode, retainDate, ErrObjectLocked
+			return mode, retainDate, legalHold, ErrObjectLocked
 		}
 		// AWS S3 just creates a new version of object when an object is being overwritten.
 		if objExists && retainDate.After(t) {
-			return mode, retainDate, ErrObjectLocked
+			return mode, retainDate, legalHold, ErrObjectLocked
+		}
+		if !legalHoldRequested {
+			// inherit retention from bucket configuration
+			return retention.Mode, objectlock.RetentionDate{Time: t.Add(retention.Validity)}, legalHold, ErrNone
 		}
-		// inherit retention from bucket configuration
-		return retention.Mode, RetentionDate{t.Add(retention.Validity)}, ErrNone
 	}
-	return mode, retainDate, ErrNone
-}
-
-// filter object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set.
-func filterObjectLockMetadata(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string, isCopy bool, getRetPerms APIErrorCode) map[string]string {
-	// Copy on write
-	dst := metadata
-	var copied bool
-	delKey := func(key string) {
-		if _, ok := metadata[key]; !ok {
-			return
-		}
-		if !copied {
-			dst = make(map[string]string, len(metadata))
-			for k, v := range metadata {
-				dst[k] = v
-			}
-			copied = true
-		}
-		delete(dst, key)
-	}
-	ret := getObjectRetentionMeta(metadata)
-	if ret.Mode == Invalid || isCopy {
-		delKey(xhttp.AmzObjectLockMode)
-		delKey(xhttp.AmzObjectLockRetainUntilDate)
-		return metadata
-	}
-	if getRetPerms == ErrNone {
-		return dst
-	}
-	delKey(xhttp.AmzObjectLockMode)
-	delKey(xhttp.AmzObjectLockRetainUntilDate)
-	return dst
+	return mode, retainDate, legalHold, ErrNone
 }

cmd/peer-rest-client.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/pkg/lifecycle"
 	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
+	"github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/policy"
 	trace "github.com/minio/minio/pkg/trace"
 )
@@ -430,7 +431,7 @@ func (client *peerRESTClient) PutBucketNotification(bucket string, rulesMap even
 }
 
 // PutBucketObjectLockConfig - PUT bucket object lock configuration.
-func (client *peerRESTClient) PutBucketObjectLockConfig(bucket string, retention Retention) error {
+func (client *peerRESTClient) PutBucketObjectLockConfig(bucket string, retention objectlock.Retention) error {
 	values := make(url.Values)
 	values.Set(peerRESTBucket, bucket)
 

cmd/peer-rest-server.go

@@ -33,6 +33,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/lifecycle"
+	objectlock "github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/policy"
 	trace "github.com/minio/minio/pkg/trace"
 )
@@ -845,7 +846,7 @@ func (s *peerRESTServer) PutBucketObjectLockConfigHandler(w http.ResponseWriter,
 		return
 	}
 
-	var retention Retention
+	var retention objectlock.Retention
 	if r.ContentLength < 0 {
 		s.writeErrorResponse(w, errInvalidArgument)
 		return

cmd/posix-errors.go

@@ -58,6 +58,11 @@ func isSysErrTooLong(err error) bool {
 	return errors.Is(err, syscall.ENAMETOOLONG)
 }
 
+// Check if the given error corresponds to the ELOOP (too many symlinks).
+func isSysErrTooManySymlinks(err error) bool {
+	return errors.Is(err, syscall.ELOOP)
+}
+
 // Check if the given error corresponds to ENOTEMPTY for unix
 // and ERROR_DIR_NOT_EMPTY for windows (directory not empty).
 func isSysErrNotEmpty(err error) bool {

cmd/posix-list-dir_unix.go

@@ -122,8 +122,11 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 		if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink {
 			fi, err := os.Stat(pathJoin(dirPath, name))
 			if err != nil {
-				// It got deleted in the meantime.
-				if os.IsNotExist(err) {
+				// It got deleted in the meantime, not found
+				// or returns too many symlinks ignore this
+				// file/directory.
+				if os.IsNotExist(err) || isSysErrPathNotFound(err) ||
+					isSysErrTooManySymlinks(err) {
 					continue
 				}
 				return nil, err

cmd/posix.go

@@ -50,8 +50,7 @@ import (
 const (
 	diskMinFreeSpace  = 900 * humanize.MiByte // Min 900MiB free space.
 	diskMinTotalSpace = diskMinFreeSpace      // Min 900MiB total space.
-	maxAllowedIOError = 5
-	readBlockSize     = 4 * humanize.MiByte // Default read block size 4MiB.
+	readBlockSize     = 4 * humanize.MiByte   // Default read block size 4MiB.
 
 	// On regular files bigger than this;
 	readAheadSize = 16 << 20
@@ -83,11 +82,10 @@ func isValidVolname(volname string) bool {
 // posix - implements StorageAPI interface.
 type posix struct {
 	// Disk usage metrics
-	totalUsed  uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
-	ioErrCount int32  // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 
-	activeIOCount    int32
 	maxActiveIOCount int32
+	activeIOCount    int32
 
 	diskPath string
 	pool     sync.Pool
@@ -219,8 +217,12 @@ func newPosix(path string) (*posix, error) {
 				return &b
 			},
 		},
-		stopUsageCh:      make(chan struct{}),
-		diskMount:        mountinfo.IsLikelyMountPoint(path),
+		stopUsageCh: make(chan struct{}),
+		diskMount:   mountinfo.IsLikelyMountPoint(path),
+		// Allow disk usage crawler to run upto 10 concurrent
+		// I/O ops, if and when activeIOCount reaches this
+		// value disk usage routine suspends the crawler
+		// and waits until activeIOCount reaches below this threshold.
 		maxActiveIOCount: 10,
 	}
 
@@ -332,91 +334,39 @@ func isQuitting(endCh chan struct{}) bool {
 }
 
 func (s *posix) waitForLowActiveIO() error {
-	t := time.NewTicker(lowActiveIOWaitTick)
-	defer t.Stop()
-	for {
-		if atomic.LoadInt32(&s.activeIOCount) >= s.maxActiveIOCount {
-			select {
-			case <-GlobalServiceDoneCh:
-				return errors.New("forced exit")
-			case <-t.C:
-				continue
-			}
+	for atomic.LoadInt32(&s.activeIOCount) >= s.maxActiveIOCount {
+		select {
+		case <-GlobalServiceDoneCh:
+			return errors.New("forced exit")
+		case <-time.NewTimer(lowActiveIOWaitTick).C:
+			continue
 		}
-		break
 	}
 	return nil
 }
 
 func (s *posix) CrawlAndGetDataUsage(endCh <-chan struct{}) (DataUsageInfo, error) {
-
-	var dataUsageInfoMu sync.Mutex
-	var dataUsageInfo = DataUsageInfo{
-		BucketsSizes:          make(map[string]uint64),
-		ObjectsSizesHistogram: make(map[string]uint64),
-	}
-
-	walkFn := func(origPath string, typ os.FileMode) error {
-
-		select {
-		case <-GlobalServiceDoneCh:
-			return filepath.SkipDir
-		default:
+	dataUsageInfo := updateUsage(s.diskPath, endCh, s.waitForLowActiveIO, func(item Item) (int64, error) {
+		// Look for `xl.json' at the leaf.
+		if !strings.HasSuffix(item.Path, SlashSeparator+xlMetaJSONFile) {
+			// if no xl.json found, skip the file.
+			return 0, errSkipFile
 		}
 
-		if err := s.waitForLowActiveIO(); err != nil {
-			return filepath.SkipDir
+		xlMetaBuf, err := ioutil.ReadFile(item.Path)
+		if err != nil {
+			return 0, errSkipFile
 		}
 
-		path := strings.TrimPrefix(origPath, s.diskPath)
-		path = strings.TrimPrefix(path, SlashSeparator)
-
-		splits := splitN(path, SlashSeparator, 2)
-
-		bucket := splits[0]
-		prefix := splits[1]
-
-		if bucket == "" {
-			return nil
+		meta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaBuf)
+		if err != nil {
+			return 0, errSkipFile
 		}
 
-		if isReservedOrInvalidBucket(bucket, false) {
-			return nil
-		}
-
-		if prefix == "" {
-			dataUsageInfoMu.Lock()
-			dataUsageInfo.BucketsCount++
-			dataUsageInfo.BucketsSizes[bucket] = 0
-			dataUsageInfoMu.Unlock()
-			return nil
-		}
-
-		if strings.HasSuffix(prefix, "/xl.json") {
-			xlMetaBuf, err := ioutil.ReadFile(origPath)
-			if err != nil {
-				return nil
-			}
-			meta, err := xlMetaV1UnmarshalJSON(context.Background(), xlMetaBuf)
-			if err != nil {
-				return nil
-			}
-
-			dataUsageInfoMu.Lock()
-			dataUsageInfo.ObjectsCount++
-			dataUsageInfo.ObjectsTotalSize += uint64(meta.Stat.Size)
-			dataUsageInfo.BucketsSizes[bucket] += uint64(meta.Stat.Size)
-			dataUsageInfo.ObjectsSizesHistogram[objSizeToHistoInterval(uint64(meta.Stat.Size))]++
-			dataUsageInfoMu.Unlock()
-		}
-
-		return nil
-	}
-
-	fastWalk(s.diskPath, walkFn)
+		return meta.Stat.Size, nil
+	})
 
 	dataUsageInfo.LastUpdate = UTCNow()
-
 	atomic.StoreUint64(&s.totalUsed, dataUsageInfo.ObjectsTotalSize)
 	return dataUsageInfo, nil
 }
@@ -434,16 +384,6 @@ type DiskInfo struct {
 // DiskInfo provides current information about disk space usage,
 // total free inodes and underlying filesystem.
 func (s *posix) DiskInfo() (info DiskInfo, err error) {
-	defer func() {
-		if s != nil && err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return info, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -557,16 +497,6 @@ func (s *posix) MakeVolBulk(volumes ...string) (err error) {
 
 // Make a volume entry.
 func (s *posix) MakeVol(volume string) (err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-
 	if !isValidVolname(volume) {
 		return errInvalidArgument
 	}
@@ -601,16 +531,6 @@ func (s *posix) MakeVol(volume string) (err error) {
 
 // ListVols - list volumes.
 func (s *posix) ListVols() (volsInfo []VolInfo, err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return nil, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -671,16 +591,6 @@ func listVols(dirPath string) ([]VolInfo, error) {
 
 // StatVol - get volume info.
 func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return VolInfo{}, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -713,16 +623,6 @@ func (s *posix) StatVol(volume string) (volInfo VolInfo, err error) {
 
 // DeleteVol - delete a volume.
 func (s *posix) DeleteVol(volume string) (err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -756,16 +656,6 @@ func (s *posix) DeleteVol(volume string) (err error) {
 func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile string,
 	readMetadataFn readMetadataFunc, endWalkCh chan struct{}) (ch chan FileInfo, err error) {
 
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return nil, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -834,16 +724,6 @@ func (s *posix) Walk(volume, dirPath, marker string, recursive bool, leafFile st
 // ListDir - return all the entries at the given directory path.
 // If an entry is a directory it will be returned with a trailing SlashSeparator.
 func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (entries []string, err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return nil, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -891,16 +771,6 @@ func (s *posix) ListDir(volume, dirPath string, count int, leafFile string) (ent
 // This API is meant to be used on files which have small memory footprint, do
 // not use this on large files as it would cause server to crash.
 func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return nil, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -963,22 +833,10 @@ func (s *posix) ReadAll(volume, path string) (buf []byte, err error) {
 // Additionally ReadFile also starts reading from an offset. ReadFile
 // semantics are same as io.ReadFull.
 func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) {
-	var n int
-	var err error
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
 	if offset < 0 {
 		return 0, errInvalidArgument
 	}
 
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return 0, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -988,6 +846,9 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 	if err != nil {
 		return 0, err
 	}
+
+	var n int
+
 	// Stat a volume entry.
 	_, err = os.Stat((volumeDir))
 	if err != nil {
@@ -1071,16 +932,6 @@ func (s *posix) ReadFile(volume, path string, offset int64, buffer []byte, verif
 }
 
 func (s *posix) openFile(volume, path string, mode int) (f *os.File, err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return nil, errFaultyDisk
-	}
-
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
 		return nil, err
@@ -1137,21 +988,10 @@ func (s *posix) openFile(volume, path string, mode int) (f *os.File, err error)
 
 // ReadFileStream - Returns the read stream of the file.
 func (s *posix) ReadFileStream(volume, path string, offset, length int64) (io.ReadCloser, error) {
-	var err error
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
 	if offset < 0 {
 		return nil, errInvalidArgument
 	}
 
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return nil, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -1230,15 +1070,6 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
 	if fileSize < -1 {
 		return errInvalidArgument
 	}
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
 
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
@@ -1334,16 +1165,6 @@ func (s *posix) CreateFile(volume, path string, fileSize int64, r io.Reader) (er
 }
 
 func (s *posix) WriteAll(volume, path string, reader io.Reader) (err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -1368,16 +1189,6 @@ func (s *posix) WriteAll(volume, path string, reader io.Reader) (err error) {
 // AppendFile - append a byte array at path, if file doesn't exist at
 // path this call explicitly creates it.
 func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -1400,16 +1211,6 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
 
 // StatFile - get file info.
 func (s *posix) StatFile(volume, path string) (file FileInfo, err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return FileInfo{}, errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -1501,16 +1302,6 @@ func deleteFile(basePath, deletePath string) error {
 
 // DeleteFile - delete a file at path.
 func (s *posix) DeleteFile(volume, path string) (err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -1555,16 +1346,6 @@ func (s *posix) DeleteFileBulk(volume string, paths []string) (errs []error, err
 
 // RenameFile - rename source path to destination path atomically.
 func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)
@@ -1652,16 +1433,6 @@ func (s *posix) RenameFile(srcVolume, srcPath, dstVolume, dstPath string) (err e
 }
 
 func (s *posix) VerifyFile(volume, path string, fileSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) (err error) {
-	defer func() {
-		if err == errFaultyDisk {
-			atomic.AddInt32(&s.ioErrCount, 1)
-		}
-	}()
-
-	if atomic.LoadInt32(&s.ioErrCount) > maxAllowedIOError {
-		return errFaultyDisk
-	}
-
 	atomic.AddInt32(&s.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt32(&s.activeIOCount, -1)

cmd/posix_test.go

@@ -310,18 +310,6 @@ func TestPosixReadAll(t *testing.T) {
 			}
 		}
 	}
-	// TestPosixing for faulty disk.
-	// Setting ioErrCount > maxAllowedIOError.
-	if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-		// setting the io error count from as specified in the test case.
-		p.storage.ioErrCount = int32(6)
-	} else {
-		t.Errorf("Expected the StorageAPI to be of type *posix")
-	}
-	_, err = posixStorage.ReadAll("abcd", "efg")
-	if err != errFaultyDisk {
-		t.Errorf("Expected err \"%s\", got err \"%s\"", errFaultyDisk, err)
-	}
 }
 
 // TestPosixNewPosix all the cases handled in posix storage layer initialization.
@@ -390,50 +378,35 @@ func TestPosixMakeVol(t *testing.T) {
 
 	testCases := []struct {
 		volName     string
-		ioErrCount  int
 		expectedErr error
 	}{
 		// TestPosix case - 1.
 		// A valid case, volume creation is expected to succeed.
 		{
 			volName:     "success-vol",
-			ioErrCount:  0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 2.
 		// Case where a file exists by the name of the volume to be created.
 		{
 			volName:     "vol-as-file",
-			ioErrCount:  0,
 			expectedErr: errVolumeExists,
 		},
 		// TestPosix case - 3.
 		{
 			volName:     "existing-vol",
-			ioErrCount:  0,
 			expectedErr: errVolumeExists,
 		},
-		// TestPosix case - 4.
-		// IO error > maxAllowedIOError, should fail with errFaultyDisk.
-		{
-			volName:     "vol",
-			ioErrCount:  6,
-			expectedErr: errFaultyDisk,
-		},
 		// TestPosix case - 5.
 		// TestPosix case with invalid volume name.
 		{
 			volName:     "ab",
-			ioErrCount:  0,
 			expectedErr: errInvalidArgument,
 		},
 	}
 
 	for i, testCase := range testCases {
-		if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-			// setting the io error count from as specified in the test case.
-			p.storage.ioErrCount = int32(testCase.ioErrCount)
-		} else {
+		if _, ok := posixStorage.(*posixDiskIDCheck); !ok {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
 		if err := posixStorage.MakeVol(testCase.volName); err != testCase.expectedErr {
@@ -503,51 +476,36 @@ func TestPosixDeleteVol(t *testing.T) {
 
 	testCases := []struct {
 		volName     string
-		ioErrCount  int
 		expectedErr error
 	}{
 		// TestPosix case  - 1.
 		// A valida case. Empty vol, should be possible to delete.
 		{
 			volName:     "success-vol",
-			ioErrCount:  0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 2.
 		// volume is non-existent.
 		{
 			volName:     "nonexistent-vol",
-			ioErrCount:  0,
 			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 3.
 		// It shouldn't be possible to delete an non-empty volume, validating the same.
 		{
 			volName:     "nonempty-vol",
-			ioErrCount:  0,
 			expectedErr: errVolumeNotEmpty,
 		},
-		// TestPosix case - 4.
-		// IO error > maxAllowedIOError, should fail with errFaultyDisk.
-		{
-			volName:     "my-disk",
-			ioErrCount:  6,
-			expectedErr: errFaultyDisk,
-		},
 		// TestPosix case - 5.
 		// Invalid volume name.
 		{
 			volName:     "ab",
-			ioErrCount:  0,
 			expectedErr: errVolumeNotFound,
 		},
 	}
 
 	for i, testCase := range testCases {
-		if posixDiskCheck, ok := posixStorage.(*posixDiskIDCheck); ok {
-			// setting the io error count from as specified in the test case.
-			posixDiskCheck.storage.ioErrCount = int32(testCase.ioErrCount)
-		} else {
+		if _, ok := posixStorage.(*posixDiskIDCheck); !ok {
 			t.Errorf("Expected the StorageAPI to be of type *posixDiskIDCheck")
 		}
 		if err = posixStorage.DeleteVol(testCase.volName); err != testCase.expectedErr {
@@ -625,42 +583,28 @@ func TestPosixStatVol(t *testing.T) {
 
 	testCases := []struct {
 		volName     string
-		ioErrCount  int
 		expectedErr error
 	}{
 		// TestPosix case - 1.
 		{
 			volName:     "success-vol",
-			ioErrCount:  0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 2.
 		{
 			volName:     "nonexistent-vol",
-			ioErrCount:  0,
 			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 3.
-		{
-			volName:     "success-vol",
-			ioErrCount:  6,
-			expectedErr: errFaultyDisk,
-		},
-		// TestPosix case - 4.
 		{
 			volName:     "ab",
-			ioErrCount:  0,
 			expectedErr: errVolumeNotFound,
 		},
 	}
 
 	for i, testCase := range testCases {
 		var volInfo VolInfo
-		// setting ioErrCnt from the test case.
-		if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-			// setting the io error count from as specified in the test case.
-			p.storage.ioErrCount = int32(testCase.ioErrCount)
-		} else {
+		if _, ok := posixStorage.(*posixDiskIDCheck); !ok {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
 		volInfo, err = posixStorage.StatVol(testCase.volName)
@@ -729,27 +673,9 @@ func TestPosixListVols(t *testing.T) {
 		t.Errorf("expected: success-vol to be created")
 	}
 
-	// setting ioErrCnt to be > maxAllowedIOError.
-	// should fail with errFaultyDisk.
-	if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-		// setting the io error count from as specified in the test case.
-		p.storage.ioErrCount = int32(6)
-	} else {
-		t.Errorf("Expected the StorageAPI to be of type *posix")
-	}
-	if _, err = posixStorage.ListVols(); err != errFaultyDisk {
-		t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errFaultyDisk, err)
-	}
 	// removing the path and simulating disk failure
 	os.RemoveAll(path)
-	// Resetting the IO error.
 	// should fail with errDiskNotFound.
-	if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-		// setting the io error count from as specified in the test case.
-		p.storage.ioErrCount = int32(0)
-	} else {
-		t.Errorf("Expected the StorageAPI to be of type *posix")
-	}
 	if _, err = posixStorage.ListVols(); err != errDiskNotFound {
 		t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errDiskNotFound, err)
 	}
@@ -783,9 +709,8 @@ func TestPosixPosixListDir(t *testing.T) {
 	}
 
 	testCases := []struct {
-		srcVol   string
-		srcPath  string
-		ioErrCnt int
+		srcVol  string
+		srcPath string
 		// expected result.
 		expectedListDir []string
 		expectedErr     error
@@ -795,7 +720,6 @@ func TestPosixPosixListDir(t *testing.T) {
 		{
 			srcVol:          "success-vol",
 			srcPath:         "abc",
-			ioErrCnt:        0,
 			expectedListDir: []string{"def/", "xyz/"},
 			expectedErr:     nil,
 		},
@@ -804,7 +728,6 @@ func TestPosixPosixListDir(t *testing.T) {
 		{
 			srcVol:          "success-vol",
 			srcPath:         "abc/def",
-			ioErrCnt:        0,
 			expectedListDir: []string{"ghi/"},
 			expectedErr:     nil,
 		},
@@ -813,7 +736,6 @@ func TestPosixPosixListDir(t *testing.T) {
 		{
 			srcVol:          "success-vol",
 			srcPath:         "abc/def/ghi",
-			ioErrCnt:        0,
 			expectedListDir: []string{"success-file"},
 			expectedErr:     nil,
 		},
@@ -821,7 +743,6 @@ func TestPosixPosixListDir(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "abcdef",
-			ioErrCnt:    0,
 			expectedErr: errFileNotFound,
 		},
 		// TestPosix case - 3.
@@ -829,34 +750,20 @@ func TestPosixPosixListDir(t *testing.T) {
 		{
 			srcVol:      "ab",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 4.
-		// TestPosix case with io error count > max limit.
-		{
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			ioErrCnt:    6,
-			expectedErr: errFaultyDisk,
-		},
-		// TestPosix case - 5.
 		// TestPosix case with non existent volume.
 		{
 			srcVol:      "non-existent-vol",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
 	}
 
 	for i, testCase := range testCases {
 		var dirList []string
-		// setting ioErrCnt from the test case.
-		if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-			// setting the io error count from as specified in the test case.
-			p.storage.ioErrCount = int32(testCase.ioErrCnt)
-		} else {
+		if _, ok := posixStorage.(*posixDiskIDCheck); !ok {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
 		dirList, err = posixStorage.ListDir(testCase.srcVol, testCase.srcPath, -1, "")
@@ -943,7 +850,6 @@ func TestPosixDeleteFile(t *testing.T) {
 	testCases := []struct {
 		srcVol      string
 		srcPath     string
-		ioErrCnt    int
 		expectedErr error
 	}{
 		// TestPosix case - 1.
@@ -951,7 +857,6 @@ func TestPosixDeleteFile(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 2.
@@ -959,58 +864,41 @@ func TestPosixDeleteFile(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: errFileNotFound,
 		},
 		// TestPosix case - 3.
-		// TestPosix case with io error count > max limit.
-		{
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			ioErrCnt:    6,
-			expectedErr: errFaultyDisk,
-		},
-		// TestPosix case - 4.
 		// TestPosix case with segment of the volume name > 255.
 		{
 			srcVol:      "my",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
-		// TestPosix case - 5.
+		// TestPosix case - 4.
 		// TestPosix case with non-existent volume.
 		{
 			srcVol:      "non-existent-vol",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
-		// TestPosix case - 6.
+		// TestPosix case - 5.
 		// TestPosix case with src path segment > 255.
 		{
 			srcVol:      "success-vol",
 			srcPath:     "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			ioErrCnt:    0,
 			expectedErr: errFileNameTooLong,
 		},
-		// TestPosix case - 7.
+		// TestPosix case - 6.
 		// TestPosix case with undeletable parent directory.
 		// File can delete, dir cannot delete because no-permissions doesn't have write perms.
 		{
 			srcVol:      "no-permissions",
 			srcPath:     "dir/file",
-			ioErrCnt:    0,
 			expectedErr: nil,
 		},
 	}
 
 	for i, testCase := range testCases {
-		// setting ioErrCnt from the test case.
-		if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-			// setting the io error count from as specified in the test case.
-			p.storage.ioErrCount = int32(testCase.ioErrCnt)
-		} else {
+		if _, ok := posixStorage.(*posixDiskIDCheck); !ok {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
 		if err = posixStorage.DeleteFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {
@@ -1242,22 +1130,6 @@ func TestPosixReadFile(t *testing.T) {
 			t.Errorf("expected: %s, got: %s", errFileAccessDenied, err)
 		}
 	}
-
-	// TestPosixing for faulty disk.
-	// setting ioErrCnt to 6.
-	// should fail with errFaultyDisk.
-	if posixType, ok := posixStorage.(*posixDiskIDCheck); ok {
-		// setting the io error count from as specified in the test case.
-		posixType.storage.ioErrCount = int32(6)
-		// Common read buffer.
-		var buf = make([]byte, 10)
-		_, err = posixType.ReadFile("abc", "yes", 0, buf, nil)
-		if err != errFaultyDisk {
-			t.Fatalf("Expected \"Faulty Disk\", got: \"%s\"", err)
-		}
-	} else {
-		t.Fatalf("Expected the StorageAPI to be of type *posixDiskIDCheck")
-	}
 }
 
 var posixReadFileWithVerifyTests = []struct {
@@ -1433,21 +1305,6 @@ func TestPosixAppendFile(t *testing.T) {
 	if err != errVolumeNotFound {
 		t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err)
 	}
-
-	// TestPosix case with IO error count > max limit.
-
-	// setting ioErrCnt to 6.
-	// should fail with errFaultyDisk.
-	if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-		// setting the io error count from as specified in the test case.
-		p.storage.ioErrCount = int32(6)
-		err = p.AppendFile("abc", "yes", []byte("hello, world"))
-		if err != errFaultyDisk {
-			t.Fatalf("Expected \"Faulty Disk\", got: \"%s\"", err)
-		}
-	} else {
-		t.Fatalf("Expected the StorageAPI to be of type *posix")
-	}
 }
 
 // TestPosix posix.RenameFile()
@@ -1494,7 +1351,6 @@ func TestPosixRenameFile(t *testing.T) {
 		destVol     string
 		srcPath     string
 		destPath    string
-		ioErrCnt    int
 		expectedErr error
 	}{
 		// TestPosix case - 1.
@@ -1503,7 +1359,6 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file1",
 			destPath:    "file-one",
-			ioErrCnt:    0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 2.
@@ -1512,7 +1367,6 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "path/",
 			destPath:    "new-path/",
-			ioErrCnt:    0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 3.
@@ -1522,7 +1376,6 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file2",
 			destPath:    "file-one",
-			ioErrCnt:    0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 4.
@@ -1533,7 +1386,6 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file3",
 			destPath:    "file-two",
-			ioErrCnt:    1,
 			expectedErr: nil,
 		},
 		// TestPosix case - 5.
@@ -1544,7 +1396,6 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file4",
 			destPath:    "file-three",
-			ioErrCnt:    5,
 			expectedErr: nil,
 		},
 		// TestPosix case - 6.
@@ -1554,7 +1405,6 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "non-existent-file",
 			destPath:    "file-three",
-			ioErrCnt:    0,
 			expectedErr: errFileNotFound,
 		},
 		// TestPosix case - 7.
@@ -1564,7 +1414,6 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "path/",
 			destPath:    "file-one",
-			ioErrCnt:    0,
 			expectedErr: errFileAccessDenied,
 		},
 		// TestPosix case - 8.
@@ -1574,20 +1423,9 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "path/",
 			destPath:    "new-path/",
-			ioErrCnt:    0,
 			expectedErr: errFileAccessDenied,
 		},
 		// TestPosix case - 9.
-		// TestPosix case with io error count is greater than maxAllowedIOError.
-		{
-			srcVol:      "src-vol",
-			destVol:     "dest-vol",
-			srcPath:     "path/",
-			destPath:    "new-path/",
-			ioErrCnt:    6,
-			expectedErr: errFaultyDisk,
-		},
-		// TestPosix case - 10.
 		// TestPosix case with source being a file and destination being a directory.
 		// Either both have to be files or directories.
 		// Expecting to fail with `errFileAccessDenied`.
@@ -1596,10 +1434,9 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file4",
 			destPath:    "new-path/",
-			ioErrCnt:    0,
 			expectedErr: errFileAccessDenied,
 		},
-		// TestPosix case - 11.
+		// TestPosix case - 10.
 		// TestPosix case with non-existent source volume.
 		// Expecting to fail with `errVolumeNotFound`.
 		{
@@ -1607,10 +1444,9 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file4",
 			destPath:    "new-path/",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
-		// TestPosix case - 12.
+		// TestPosix case - 11.
 		// TestPosix case with non-existent destination volume.
 		// Expecting to fail with `errVolumeNotFound`.
 		{
@@ -1618,10 +1454,9 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol-non-existent",
 			srcPath:     "file4",
 			destPath:    "new-path/",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
-		// TestPosix case - 13.
+		// TestPosix case - 12.
 		// TestPosix case with invalid src volume name. Length should be atleast 3.
 		// Expecting to fail with `errInvalidArgument`.
 		{
@@ -1629,7 +1464,16 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol-non-existent",
 			srcPath:     "file4",
 			destPath:    "new-path/",
-			ioErrCnt:    0,
+			expectedErr: errVolumeNotFound,
+		},
+		// TestPosix case - 13.
+		// TestPosix case with invalid destination volume name. Length should be atleast 3.
+		// Expecting to fail with `errInvalidArgument`.
+		{
+			srcVol:      "abcd",
+			destVol:     "ef",
+			srcPath:     "file4",
+			destPath:    "new-path/",
 			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 14.
@@ -1640,21 +1484,9 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "ef",
 			srcPath:     "file4",
 			destPath:    "new-path/",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
 		// TestPosix case - 15.
-		// TestPosix case with invalid destination volume name. Length should be atleast 3.
-		// Expecting to fail with `errInvalidArgument`.
-		{
-			srcVol:      "abcd",
-			destVol:     "ef",
-			srcPath:     "file4",
-			destPath:    "new-path/",
-			ioErrCnt:    0,
-			expectedErr: errVolumeNotFound,
-		},
-		// TestPosix case - 16.
 		// TestPosix case with the parent of the destination being a file.
 		// expected to fail with `errFileAccessDenied`.
 		{
@@ -1662,10 +1494,9 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file5",
 			destPath:    "file-one/parent-is-file",
-			ioErrCnt:    0,
 			expectedErr: errFileAccessDenied,
 		},
-		// TestPosix case - 17.
+		// TestPosix case - 16.
 		// TestPosix case with segment of source file name more than 255.
 		// expected not to fail.
 		{
@@ -1673,10 +1504,9 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
 			destPath:    "file-six",
-			ioErrCnt:    0,
 			expectedErr: errFileNameTooLong,
 		},
-		// TestPosix case - 18.
+		// TestPosix case - 17.
 		// TestPosix case with segment of destination file name more than 255.
 		// expected not to fail.
 		{
@@ -1684,17 +1514,12 @@ func TestPosixRenameFile(t *testing.T) {
 			destVol:     "dest-vol",
 			srcPath:     "file6",
 			destPath:    "path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001",
-			ioErrCnt:    0,
 			expectedErr: errFileNameTooLong,
 		},
 	}
 
 	for i, testCase := range testCases {
-		// setting ioErrCnt from the test case.
-		if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-			// setting the io error count from as specified in the test case.
-			p.storage.ioErrCount = int32(testCase.ioErrCnt)
-		} else {
+		if _, ok := posixStorage.(*posixDiskIDCheck); !ok {
 			t.Fatalf("Expected the StorageAPI to be of type *posix")
 		}
 
@@ -1729,7 +1554,6 @@ func TestPosixStatFile(t *testing.T) {
 	testCases := []struct {
 		srcVol      string
 		srcPath     string
-		ioErrCnt    int
 		expectedErr error
 	}{
 		// TestPosix case - 1.
@@ -1737,7 +1561,6 @@ func TestPosixStatFile(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 2.
@@ -1745,7 +1568,6 @@ func TestPosixStatFile(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "path/to/success-file",
-			ioErrCnt:    0,
 			expectedErr: nil,
 		},
 		// TestPosix case - 3.
@@ -1753,7 +1575,6 @@ func TestPosixStatFile(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "nonexistent-file",
-			ioErrCnt:    0,
 			expectedErr: errFileNotFound,
 		},
 		// TestPosix case - 4.
@@ -1761,7 +1582,6 @@ func TestPosixStatFile(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "path/2/success-file",
-			ioErrCnt:    0,
 			expectedErr: errFileNotFound,
 		},
 		// TestPosix case - 5.
@@ -1769,33 +1589,19 @@ func TestPosixStatFile(t *testing.T) {
 		{
 			srcVol:      "success-vol",
 			srcPath:     "path",
-			ioErrCnt:    0,
 			expectedErr: errFileNotFound,
 		},
 		// TestPosix case - 6.
-		// TestPosix case with io error count > max limit.
-		{
-			srcVol:      "success-vol",
-			srcPath:     "success-file",
-			ioErrCnt:    6,
-			expectedErr: errFaultyDisk,
-		},
-		// TestPosix case - 7.
 		// TestPosix case with non existent volume.
 		{
 			srcVol:      "non-existent-vol",
 			srcPath:     "success-file",
-			ioErrCnt:    0,
 			expectedErr: errVolumeNotFound,
 		},
 	}
 
 	for i, testCase := range testCases {
-		// setting ioErrCnt from the test case.
-		if p, ok := posixStorage.(*posixDiskIDCheck); ok {
-			// setting the io error count from as specified in the test case.
-			p.storage.ioErrCount = int32(testCase.ioErrCnt)
-		} else {
+		if _, ok := posixStorage.(*posixDiskIDCheck); !ok {
 			t.Errorf("Expected the StorageAPI to be of type *posix")
 		}
 		if _, err := posixStorage.StatFile(testCase.srcVol, testCase.srcPath); err != testCase.expectedErr {

cmd/prepare-storage.go

@@ -207,6 +207,7 @@ func IsServerResolvable(endpoint Endpoint) error {
 
 	resp, err := httpClient.Do(req)
 	if err != nil {
+		httpClient.CloseIdleConnections()
 		return err
 	}
 	defer xhttp.DrainBody(resp.Body)
@@ -220,7 +221,7 @@ func IsServerResolvable(endpoint Endpoint) error {
 // connect to list of endpoints and load all XL disk formats, validate the formats are correct
 // and are in quorum, if no formats are found attempt to initialize all of them for the first
 // time. additionally make sure to close all the disks used in this attempt.
-func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, setCount, drivesPerSet int, deploymentID string) (*formatXLV3, error) {
+func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) (*formatXLV3, error) {
 	// Initialize all storage disks
 	storageDisks, errs := initStorageDisksWithErrors(endpoints)
 	defer closeStorageDisks(storageDisks)
@@ -256,6 +257,9 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 
 	// All disks report unformatted we should initialized everyone.
 	if shouldInitXLDisks(sErrs) && firstDisk {
+		logger.Info("Formatting %v zone, %v set(s), %v drives per set.",
+			zoneCount, setCount, drivesPerSet)
+
 		// Initialize erasure code format on disks
 		format, err := initFormatXL(context.Background(), storageDisks, setCount, drivesPerSet, deploymentID)
 		if err != nil {
@@ -333,7 +337,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 }
 
 // Format disks before initialization of object layer.
-func waitForFormatXL(firstDisk bool, endpoints Endpoints, setCount, drivesPerSet int, deploymentID string) (format *formatXLV3, err error) {
+func waitForFormatXL(firstDisk bool, endpoints Endpoints, zoneCount, setCount, drivesPerSet int, deploymentID string) (format *formatXLV3, err error) {
 	if len(endpoints) == 0 || setCount == 0 || drivesPerSet == 0 {
 		return nil, errInvalidArgument
 	}
@@ -360,7 +364,7 @@ func waitForFormatXL(firstDisk bool, endpoints Endpoints, setCount, drivesPerSet
 	for {
 		select {
 		case <-ticker.C:
-			format, err := connectLoadInitFormats(tries, firstDisk, endpoints, setCount, drivesPerSet, deploymentID)
+			format, err := connectLoadInitFormats(tries, firstDisk, endpoints, zoneCount, setCount, drivesPerSet, deploymentID)
 			if err != nil {
 				tries++
 				switch err {

cmd/rest/client.go

@@ -69,6 +69,7 @@ func (c *Client) CallWithContext(ctx context.Context, method string, values url.
 	}
 	resp, err := c.httpClient.Do(req)
 	if err != nil {
+		c.httpClient.CloseIdleConnections()
 		return nil, &NetworkError{err}
 	}
 

cmd/server-main.go

@@ -255,6 +255,7 @@ func serverMain(ctx *cli.Context) {
 	if ctx.Args().First() == "help" || !endpointsPresent(ctx) {
 		cli.ShowCommandHelpAndExit(ctx, "server", 1)
 	}
+	setDefaultProfilerRates()
 
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background())

cmd/signals.go

@@ -30,10 +30,8 @@ func handleSignals() {
 		// If global profiler is set stop before we exit.
 		globalProfilerMu.Lock()
 		defer globalProfilerMu.Unlock()
-		if len(globalProfiler) > 0 {
-			for _, p := range globalProfiler {
-				p.Stop()
-			}
+		for _, p := range globalProfiler {
+			p.Stop()
 		}
 
 		if success {

cmd/signature-v4.go

@@ -222,14 +222,6 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 		return errCode
 	}
 
-	// Construct new query.
-	query := make(url.Values)
-	if req.URL.Query().Get(xhttp.AmzContentSha256) != "" {
-		query.Set(xhttp.AmzContentSha256, hashedPayload)
-	}
-
-	query.Set(xhttp.AmzAlgorithm, signV4Algorithm)
-
 	// If the host which signed the request is slightly ahead in time (by less than globalMaxSkewTime) the
 	// request should still be allowed.
 	if pSignValues.Date.After(UTCNow().Add(globalMaxSkewTime)) {
@@ -244,6 +236,20 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 	t := pSignValues.Date
 	expireSeconds := int(pSignValues.Expires / time.Second)
 
+	// Construct new query.
+	query := make(url.Values)
+	clntHashedPayload := req.URL.Query().Get(xhttp.AmzContentSha256)
+	if clntHashedPayload != "" {
+		query.Set(xhttp.AmzContentSha256, hashedPayload)
+	}
+
+	token := req.URL.Query().Get(xhttp.AmzSecurityToken)
+	if token != "" {
+		query.Set(xhttp.AmzSecurityToken, cred.SessionToken)
+	}
+
+	query.Set(xhttp.AmzAlgorithm, signV4Algorithm)
+
 	// Construct the query.
 	query.Set(xhttp.AmzDate, t.Format(iso8601Format))
 	query.Set(xhttp.AmzExpires, strconv.Itoa(expireSeconds))
@@ -262,6 +268,7 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 
 		if strings.Contains(key, "x-amz-server-side-") {
 			query.Set(k, v[0])
+			continue
 		}
 
 		if strings.HasPrefix(key, "x-amz") {
@@ -290,10 +297,12 @@ func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region s
 		return ErrSignatureDoesNotMatch
 	}
 	// Verify if sha256 payload query is same.
-	if req.URL.Query().Get(xhttp.AmzContentSha256) != "" {
-		if req.URL.Query().Get(xhttp.AmzContentSha256) != query.Get(xhttp.AmzContentSha256) {
-			return ErrContentSHA256Mismatch
-		}
+	if clntHashedPayload != "" && clntHashedPayload != query.Get(xhttp.AmzContentSha256) {
+		return ErrContentSHA256Mismatch
+	}
+	// Verify if security token is correct.
+	if token != "" && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+		return ErrInvalidToken
 	}
 
 	/// Verify finally if signature is same.

cmd/storage-rest-server.go

@@ -119,15 +119,9 @@ func (s *storageRESTServer) CrawlAndGetDataUsageHandler(w http.ResponseWriter, r
 		return
 	}
 
-	usageInfo, err := s.storage.CrawlAndGetDataUsage(GlobalServiceDoneCh)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
 	w.Header().Set(xhttp.ContentType, "text/event-stream")
 	doneCh := sendWhiteSpaceToHTTPResponse(w)
-	usageInfo, err = s.storage.CrawlAndGetDataUsage(GlobalServiceDoneCh)
+	usageInfo, err := s.storage.CrawlAndGetDataUsage(GlobalServiceDoneCh)
 	<-doneCh
 
 	if err != nil {

cmd/test-utils_test.go

@@ -193,7 +193,7 @@ func prepareXLSets32() (ObjectLayer, []string, error) {
 
 	endpoints := append(endpoints1, endpoints2...)
 	fsDirs := append(fsDirs1, fsDirs2...)
-	format, err := waitForFormatXL(true, endpoints, 2, 16, "")
+	format, err := waitForFormatXL(true, endpoints, 1, 2, 16, "")
 	if err != nil {
 		removeRoots(fsDirs)
 		return nil, nil, err

cmd/utils.go

@@ -39,10 +39,10 @@ import (
 	"sync"
 	"time"
 
-	"github.com/beevik/ntp"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
+	"github.com/minio/minio/pkg/madmin"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
@@ -68,28 +68,24 @@ func request2BucketObjectName(r *http.Request) (bucketName, objectName string) {
 	if err != nil {
 		logger.CriticalIf(context.Background(), err)
 	}
-	return urlPath2BucketObjectName(path)
+
+	return path2BucketObject(path)
 }
 
-// Convert url path into bucket and object name.
-func urlPath2BucketObjectName(path string) (bucketName, objectName string) {
-	if path == "" || path == SlashSeparator {
-		return "", ""
+// path2BucketObjectWithBasePath returns bucket and prefix, if any,
+// of a 'path'. basePath is trimmed from the front of the 'path'.
+func path2BucketObjectWithBasePath(basePath, path string) (bucket, prefix string) {
+	path = strings.TrimPrefix(path, basePath)
+	path = strings.TrimPrefix(path, SlashSeparator)
+	m := strings.Index(path, SlashSeparator)
+	if m < 0 {
+		return path, ""
 	}
+	return path[:m], path[m+len(SlashSeparator):]
+}
 
-	// Trim any preceding slash separator.
-	urlPath := strings.TrimPrefix(path, SlashSeparator)
-
-	// Split urlpath using slash separator into a given number of
-	// expected tokens.
-	tokens := strings.SplitN(urlPath, SlashSeparator, 2)
-	bucketName = tokens[0]
-	if len(tokens) == 2 {
-		objectName = tokens[1]
-	}
-
-	// Success.
-	return bucketName, objectName
+func path2BucketObject(s string) (bucket, prefix string) {
+	return path2BucketObjectWithBasePath("", s)
 }
 
 // URI scheme constants.
@@ -190,9 +186,28 @@ func contains(slice interface{}, elem interface{}) bool {
 // provide any API to calculate the profiler file path in the
 // disk since the name of this latter is randomly generated.
 type profilerWrapper struct {
+	// Profile recorded at start of benchmark.
+	base   []byte
 	stopFn func() ([]byte, error)
 }
 
+// recordBase will record the profile and store it as the base.
+func (p *profilerWrapper) recordBase(name string) {
+	var buf bytes.Buffer
+	p.base = nil
+	err := pprof.Lookup(name).WriteTo(&buf, 0)
+	if err != nil {
+		return
+	}
+	p.base = buf.Bytes()
+}
+
+// Base returns the recorded base if any.
+func (p profilerWrapper) Base() []byte {
+	return p.base
+}
+
+// Stop the currently running benchmark.
 func (p profilerWrapper) Stop() ([]byte, error) {
 	return p.stopFn()
 }
@@ -216,18 +231,28 @@ func getProfileData() (map[string][]byte, error) {
 		if err == nil {
 			dst[typ] = buf
 		}
+		buf = prof.Base()
+		if len(buf) > 0 {
+			dst[typ+"-before"] = buf
+		}
 	}
 	return dst, nil
 }
 
+func setDefaultProfilerRates() {
+	runtime.MemProfileRate = 4096      // 512K -> 4K - Must be constant throughout application lifetime.
+	runtime.SetMutexProfileFraction(0) // Disable until needed
+	runtime.SetBlockProfileRate(0)     // Disable until needed
+}
+
 // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
 func startProfiler(profilerType string) (minioProfiler, error) {
 	var prof profilerWrapper
 
 	// Enable profiler and set the name of the file that pkg/pprof
 	// library creates to store profiling data.
-	switch profilerType {
-	case "cpu":
+	switch madmin.ProfilerType(profilerType) {
+	case madmin.ProfilerCPU:
 		dirPath, err := ioutil.TempDir("", "profile")
 		if err != nil {
 			return nil, err
@@ -250,32 +275,41 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 			defer os.RemoveAll(dirPath)
 			return ioutil.ReadFile(fn)
 		}
-	case "mem":
-		old := runtime.MemProfileRate
-		runtime.MemProfileRate = 4096
+	case madmin.ProfilerMEM:
+		runtime.GC()
+		prof.recordBase("heap")
 		prof.stopFn = func() ([]byte, error) {
+			runtime.GC()
 			var buf bytes.Buffer
-			runtime.MemProfileRate = old
 			err := pprof.Lookup("heap").WriteTo(&buf, 0)
 			return buf.Bytes(), err
 		}
-	case "block":
+	case madmin.ProfilerBlock:
+		prof.recordBase("block")
 		runtime.SetBlockProfileRate(1)
 		prof.stopFn = func() ([]byte, error) {
 			var buf bytes.Buffer
-			runtime.SetBlockProfileRate(0)
 			err := pprof.Lookup("block").WriteTo(&buf, 0)
+			runtime.SetBlockProfileRate(0)
 			return buf.Bytes(), err
 		}
-	case "mutex":
+	case madmin.ProfilerMutex:
+		prof.recordBase("mutex")
 		runtime.SetMutexProfileFraction(1)
 		prof.stopFn = func() ([]byte, error) {
 			var buf bytes.Buffer
-			runtime.SetMutexProfileFraction(0)
 			err := pprof.Lookup("mutex").WriteTo(&buf, 0)
+			runtime.SetMutexProfileFraction(0)
 			return buf.Bytes(), err
 		}
-	case "trace":
+	case madmin.ProfilerThreads:
+		prof.recordBase("threadcreate")
+		prof.stopFn = func() ([]byte, error) {
+			var buf bytes.Buffer
+			err := pprof.Lookup("threadcreate").WriteTo(&buf, 0)
+			return buf.Bytes(), err
+		}
+	case madmin.ProfilerTrace:
 		dirPath, err := ioutil.TempDir("", "profile")
 		if err != nil {
 			return nil, err
@@ -307,6 +341,8 @@ func startProfiler(profilerType string) (minioProfiler, error) {
 
 // minioProfiler - minio profiler interface.
 type minioProfiler interface {
+	// Return base profile. 'nil' if none.
+	Base() []byte
 	// Stop the profiler
 	Stop() ([]byte, error)
 }
@@ -354,17 +390,6 @@ func UTCNow() time.Time {
 	return time.Now().UTC()
 }
 
-// UTCNowNTP - is similar in functionality to UTCNow()
-// but only used when we do not wish to rely on system
-// time.
-func UTCNowNTP() (time.Time, error) {
-	// ntp server is disabled
-	if ntpServer == "" {
-		return UTCNow(), nil
-	}
-	return ntp.Time(ntpServer)
-}
-
 // GenETag - generate UUID based ETag
 func GenETag() string {
 	return ToS3ETag(getMD5Hash([]byte(mustGetUUID())))
@@ -565,16 +590,6 @@ func getMinioMode() string {
 	return mode
 }
 
-func splitN(str, delim string, num int) []string {
-	stdSplit := strings.SplitN(str, delim, num)
-	retSplit := make([]string, num)
-	for i := 0; i < len(stdSplit); i++ {
-		retSplit[i] = stdSplit[i]
-	}
-
-	return retSplit
-}
-
 func iamPolicyClaimName() string {
 	return globalOpenIDConfig.ClaimPrefix + globalOpenIDConfig.ClaimName
 }

cmd/utils_test.go

@@ -107,85 +107,74 @@ func TestMaxPartID(t *testing.T) {
 	}
 }
 
-// Tests extracting bucket and objectname from various types of URL paths.
-func TestURL2BucketObjectName(t *testing.T) {
+// Tests extracting bucket and objectname from various types of paths.
+func TestPath2BucketObjectName(t *testing.T) {
 	testCases := []struct {
-		u              *url.URL
+		path           string
 		bucket, object string
 	}{
 		// Test case 1 normal case.
 		{
-			u: &url.URL{
-				Path: "/bucket/object",
-			},
+			path:   "/bucket/object",
 			bucket: "bucket",
 			object: "object",
 		},
 		// Test case 2 where url only has separator.
 		{
-			u: &url.URL{
-				Path: SlashSeparator,
-			},
+			path:   SlashSeparator,
 			bucket: "",
 			object: "",
 		},
 		// Test case 3 only bucket is present.
 		{
-			u: &url.URL{
-				Path: "/bucket",
-			},
+			path:   "/bucket",
 			bucket: "bucket",
 			object: "",
 		},
 		// Test case 4 many separators and object is a directory.
 		{
-			u: &url.URL{
-				Path: "/bucket/object/1/",
-			},
+			path:   "/bucket/object/1/",
 			bucket: "bucket",
 			object: "object/1/",
 		},
 		// Test case 5 object has many trailing separators.
 		{
-			u: &url.URL{
-				Path: "/bucket/object/1///",
-			},
+			path:   "/bucket/object/1///",
 			bucket: "bucket",
 			object: "object/1///",
 		},
 		// Test case 6 object has only trailing separators.
 		{
-			u: &url.URL{
-				Path: "/bucket/object///////",
-			},
+			path:   "/bucket/object///////",
 			bucket: "bucket",
 			object: "object///////",
 		},
 		// Test case 7 object has preceding separators.
 		{
-			u: &url.URL{
-				Path: "/bucket////object////",
-			},
+			path:   "/bucket////object////",
 			bucket: "bucket",
 			object: "///object////",
 		},
-		// Test case 9 url path is empty.
+		// Test case 8 url path is empty.
 		{
-			u:      &url.URL{},
+			path:   "",
 			bucket: "",
 			object: "",
 		},
 	}
 
 	// Validate all test cases.
-	for i, testCase := range testCases {
-		bucketName, objectName := urlPath2BucketObjectName(testCase.u.Path)
-		if bucketName != testCase.bucket {
-			t.Errorf("Test %d: failed expected bucket name \"%s\", got \"%s\"", i+1, testCase.bucket, bucketName)
-		}
-		if objectName != testCase.object {
-			t.Errorf("Test %d: failed expected bucket name \"%s\", got \"%s\"", i+1, testCase.object, objectName)
-		}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			bucketName, objectName := path2BucketObject(testCase.path)
+			if bucketName != testCase.bucket {
+				t.Errorf("failed expected bucket name \"%s\", got \"%s\"", testCase.bucket, bucketName)
+			}
+			if objectName != testCase.object {
+				t.Errorf("failed expected bucket name \"%s\", got \"%s\"", testCase.object, objectName)
+			}
+		})
 	}
 }
 

cmd/web-handlers.go

@@ -50,6 +50,7 @@ import (
 	"github.com/minio/minio/pkg/hash"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/ioutil"
+	"github.com/minio/minio/pkg/objectlock"
 	"github.com/minio/minio/pkg/policy"
 )
 
@@ -928,6 +929,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 	object := vars["object"]
 
 	retPerms := ErrAccessDenied
+	holdPerms := ErrAccessDenied
 
 	claims, owner, authErr := webRequestAuthenticate(r)
 	if authErr != nil {
@@ -974,7 +976,17 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}) {
 			retPerms = ErrNone
 		}
-
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.AccessKey(),
+			Action:          iampolicy.PutObjectLegalHoldAction,
+			BucketName:      bucket,
+			ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
+			IsOwner:         owner,
+			ObjectName:      object,
+			Claims:          claims.Map(),
+		}) {
+			holdPerms = ErrNone
+		}
 	}
 
 	// Check if bucket is a reserved bucket name or invalid.
@@ -1068,11 +1080,14 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		getObjectInfo = web.CacheAPI().GetObjectInfo
 	}
 	// enforce object retention rules
-	retentionMode, retentionDate, s3Err := checkPutObjectRetentionAllowed(ctx, r, bucket, object, getObjectInfo, retPerms)
+	retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
 	if s3Err == ErrNone && retentionMode != "" {
 		opts.UserDefined[xhttp.AmzObjectLockMode] = string(retentionMode)
 		opts.UserDefined[xhttp.AmzObjectLockRetainUntilDate] = retentionDate.UTC().Format(time.RFC3339)
 	}
+	if s3Err == ErrNone && legalHold.Status != "" {
+		opts.UserDefined[xhttp.AmzObjectLockLegalHold] = string(legalHold.Status)
+	}
 	if s3Err != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL, guessIsBrowserReq(r))
 		return
@@ -1130,6 +1145,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	token := r.URL.Query().Get("token")
 
 	getRetPerms := ErrAccessDenied
+	legalHoldPerms := ErrAccessDenied
 
 	claims, owner, authErr := webTokenAuthenticate(token)
 	if authErr != nil {
@@ -1154,6 +1170,15 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 			}) {
 				getRetPerms = ErrNone
 			}
+			if globalPolicySys.IsAllowed(policy.Args{
+				Action:          policy.GetObjectLegalHoldAction,
+				BucketName:      bucket,
+				ConditionValues: getConditionValues(r, "", "", nil),
+				IsOwner:         false,
+				ObjectName:      object,
+			}) {
+				legalHoldPerms = ErrNone
+			}
 		} else {
 			writeWebErrorResponse(w, authErr)
 			return
@@ -1185,6 +1210,17 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 		}) {
 			getRetPerms = ErrNone
 		}
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.AccessKey(),
+			Action:          iampolicy.GetObjectLegalHoldAction,
+			BucketName:      bucket,
+			ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
+			IsOwner:         owner,
+			ObjectName:      object,
+			Claims:          claims.Map(),
+		}) {
+			legalHoldPerms = ErrNone
+		}
 	}
 
 	// Check if bucket is a reserved bucket name or invalid.
@@ -1209,7 +1245,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
 	objInfo := gr.ObjInfo
 
 	// filter object lock metadata if permission does not permit
-	objInfo.UserDefined = filterObjectLockMetadata(ctx, r, bucket, object, objInfo.UserDefined, false, getRetPerms)
+	objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone)
 
 	if objectAPI.IsEncryptionSupported() {
 		if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
@@ -1304,6 +1340,8 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 	token := r.URL.Query().Get("token")
 	claims, owner, authErr := webTokenAuthenticate(token)
 	var getRetPerms []APIErrorCode
+	var legalHoldPerms []APIErrorCode
+
 	if authErr != nil {
 		if authErr == errNoAuthToken {
 			for _, object := range args.Objects {
@@ -1329,6 +1367,18 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 					retentionPerm = ErrNone
 				}
 				getRetPerms = append(getRetPerms, retentionPerm)
+
+				legalHoldPerm := ErrAccessDenied
+				if globalPolicySys.IsAllowed(policy.Args{
+					Action:          policy.GetObjectLegalHoldAction,
+					BucketName:      args.BucketName,
+					ConditionValues: getConditionValues(r, "", "", nil),
+					IsOwner:         false,
+					ObjectName:      pathJoin(args.Prefix, object),
+				}) {
+					legalHoldPerm = ErrNone
+				}
+				legalHoldPerms = append(legalHoldPerms, legalHoldPerm)
 			}
 		} else {
 			writeWebErrorResponse(w, authErr)
@@ -1354,7 +1404,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 			retentionPerm := ErrAccessDenied
 			if globalIAMSys.IsAllowed(iampolicy.Args{
 				AccountName:     claims.AccessKey(),
-				Action:          iampolicy.GetObjectAction,
+				Action:          iampolicy.GetObjectRetentionAction,
 				BucketName:      args.BucketName,
 				ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
 				IsOwner:         owner,
@@ -1364,6 +1414,20 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 				retentionPerm = ErrNone
 			}
 			getRetPerms = append(getRetPerms, retentionPerm)
+
+			legalHoldPerm := ErrAccessDenied
+			if globalIAMSys.IsAllowed(iampolicy.Args{
+				AccountName:     claims.AccessKey(),
+				Action:          iampolicy.GetObjectLegalHoldAction,
+				BucketName:      args.BucketName,
+				ConditionValues: getConditionValues(r, "", claims.AccessKey(), claims.Map()),
+				IsOwner:         owner,
+				ObjectName:      pathJoin(args.Prefix, object),
+				Claims:          claims.Map(),
+			}) {
+				legalHoldPerm = ErrNone
+			}
+			legalHoldPerms = append(legalHoldPerms, legalHoldPerm)
 		}
 	}
 
@@ -1394,7 +1458,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
 
 			info := gr.ObjInfo
 			// filter object lock metadata if permission does not permit
-			info.UserDefined = filterObjectLockMetadata(ctx, r, args.BucketName, objectName, info.UserDefined, false, getRetPerms[i])
+			info.UserDefined = objectlock.FilterObjectLockMetadata(info.UserDefined, getRetPerms[i] != ErrNone, legalHoldPerms[i] != ErrNone)
 
 			if info.IsCompressed() {
 				// For reporting, set the file size to the uncompressed size.
@@ -1659,7 +1723,7 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB
 
 	reply.UIVersion = browser.UIVersion
 	for prefix, policy := range miniogopolicy.GetPolicies(policyInfo.Statements, args.BucketName, "") {
-		bucketName, objectPrefix := urlPath2BucketObjectName(prefix)
+		bucketName, objectPrefix := path2BucketObject(prefix)
 		objectPrefix = strings.TrimSuffix(objectPrefix, "*")
 		reply.Policies = append(reply.Policies, BucketAccessPolicy{
 			Bucket: bucketName,
@@ -1965,6 +2029,7 @@ func (web *webAPIHandlers) LoginSTS(r *http.Request, args *LoginSTSArgs, reply *
 
 	resp, err := clnt.Do(req)
 	if err != nil {
+		clnt.CloseIdleConnections()
 		return toJSONError(ctx, err)
 	}
 	defer xhttp.DrainBody(resp.Body)

cmd/web-handlers_test.go

@@ -1590,12 +1590,14 @@ func TestWebObjectLayerFaultyDisks(t *testing.T) {
 	z := obj.(*xlZones)
 	xl := z.zones[0].sets[0]
 	xlDisks := xl.getDisks()
+	z.zones[0].xlDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		for i, d := range xlDisks {
 			xlDisks[i] = newNaughtyDisk(d, nil, errFaultyDisk)
 		}
 		return xlDisks
 	}
+	z.zones[0].xlDisksMu.Unlock()
 
 	// Initialize web rpc endpoint.
 	apiRouter := initTestWebRPCEndPoint(obj)

cmd/xl-sets.go

@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 // setsStorageAPI is encapsulated type for Close()
@@ -434,6 +435,10 @@ func (s *xlSets) StorageInfo(ctx context.Context) StorageInfo {
 	return storageInfo
 }
 
+func (s *xlSets) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+	return DataUsageInfo{}
+}
+
 // Shutdown shutsdown all erasure coded sets in parallel
 // returns error upon first error.
 func (s *xlSets) Shutdown(ctx context.Context) error {
@@ -1661,6 +1666,21 @@ func (s *xlSets) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, de
 	return s.listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys, true)
 }
 
+// PutObjectTag - replace or add tags to an existing object
+func (s *xlSets) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	return s.getHashedSet(object).PutObjectTag(ctx, bucket, object, tags)
+}
+
+// DeleteObjectTag - delete object tags from an existing object
+func (s *xlSets) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	return s.getHashedSet(object).DeleteObjectTag(ctx, bucket, object)
+}
+
+// GetObjectTag - get object tags from an existing object
+func (s *xlSets) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	return s.getHashedSet(object).GetObjectTag(ctx, bucket, object)
+}
+
 // GetMetrics - no op
 func (s *xlSets) GetMetrics(ctx context.Context) (*Metrics, error) {
 	logger.LogIf(ctx, NotImplemented{})

cmd/xl-sets_test.go

@@ -76,18 +76,18 @@ func TestNewXLSets(t *testing.T) {
 	}
 
 	endpoints := mustGetNewEndpoints(erasureDisks...)
-	_, err := waitForFormatXL(true, endpoints, 0, 16, "")
+	_, err := waitForFormatXL(true, endpoints, 1, 0, 16, "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)
 	}
 
-	_, err = waitForFormatXL(true, nil, 1, 16, "")
+	_, err = waitForFormatXL(true, nil, 1, 1, 16, "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)
 	}
 
 	// Initializes all erasure disks
-	format, err := waitForFormatXL(true, endpoints, 1, 16, "")
+	format, err := waitForFormatXL(true, endpoints, 1, 1, 16, "")
 	if err != nil {
 		t.Fatalf("Unable to format disks for erasure, %s", err)
 	}

cmd/xl-v1-healing.go

@@ -92,11 +92,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, bucket string, w
 					return nil
 				}
 
-				makeErr := storageDisks[index].MakeVol(bucket)
-				if makeErr == nil {
-					afterState[index] = madmin.DriveStateOk
-				}
-				return makeErr
+				return serr
 			}
 			beforeState[index] = madmin.DriveStateOk
 			afterState[index] = madmin.DriveStateOk
@@ -106,12 +102,18 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, bucket string, w
 
 	errs := g.Wait()
 
+	reducedErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum-1)
+	if reducedErr == errVolumeNotFound {
+		return res, nil
+	}
+
 	// Initialize heal result info
 	res = madmin.HealResultItem{
 		Type:      madmin.HealItemBucket,
 		Bucket:    bucket,
 		DiskCount: len(storageDisks),
 	}
+
 	for i := range beforeState {
 		if storageDisks[i] != nil {
 			drive := storageDisks[i].String()
@@ -128,11 +130,32 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, bucket string, w
 		}
 	}
 
-	reducedErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum)
+	// Initialize sync waitgroup.
+	g = errgroup.WithNErrs(len(storageDisks))
+
+	// Make a volume entry on all underlying storage disks.
+	for index := range storageDisks {
+		index := index
+		g.Go(func() error {
+			if beforeState[index] == madmin.DriveStateMissing {
+				makeErr := storageDisks[index].MakeVol(bucket)
+				if makeErr == nil {
+					afterState[index] = madmin.DriveStateOk
+				}
+				return makeErr
+			}
+			return errs[index]
+		}, index)
+	}
+
+	errs = g.Wait()
+
+	reducedErr = reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum)
 	if reducedErr == errXLWriteQuorum {
 		// Purge successfully created buckets if we don't have writeQuorum.
 		undoMakeBucket(storageDisks, bucket)
 	}
+
 	return res, reducedErr
 }
 

cmd/xl-v1-healing_test.go

@@ -270,6 +270,7 @@ func TestHealObjectXL(t *testing.T) {
 	}
 
 	xlDisks := xl.getDisks()
+	z.zones[0].xlDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		// Nil more than half the disks, to remove write quorum.
 		for i := 0; i <= len(xlDisks)/2; i++ {
@@ -277,6 +278,7 @@ func TestHealObjectXL(t *testing.T) {
 		}
 		return xlDisks
 	}
+	z.zones[0].xlDisksMu.Unlock()
 
 	// Try healing now, expect to receive errDiskNotFound.
 	_, err = obj.HealObject(context.Background(), bucket, object, false, false, madmin.HealDeepScan)

cmd/xl-v1-metadata.go

@@ -245,9 +245,13 @@ func (m xlMetaV1) ToObjectInfo(bucket, object string) ObjectInfo {
 	// Extract etag from metadata.
 	objInfo.ETag = extractETag(m.Meta)
 
+	// Add user tags to the object info
+	objInfo.UserTags = m.Meta[xhttp.AmzObjectTagging]
+
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of
 	// response headers. e.g, X-Minio-* or X-Amz-*.
+	// Tags have also been extracted, we remove that as well.
 	objInfo.UserDefined = cleanMetadata(m.Meta)
 
 	// All the parts per object.

cmd/xl-v1-multipart.go

@@ -353,7 +353,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 		defer xl.bp.Put(buffer)
 	case size < blockSizeV1:
 		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
-		buffer = make([]byte, size, 2*size)
+		buffer = make([]byte, size, 2*size+int64(erasure.parityBlocks+erasure.dataBlocks-1))
 	}
 
 	if len(buffer) > int(xlMeta.Erasure.BlockSize) {

cmd/xl-v1-object.go

@@ -27,6 +27,7 @@ import (
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/mimedb"
 	"github.com/minio/minio/pkg/sync/errgroup"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 // list all errors which can be ignored in object operations.
@@ -559,7 +560,7 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
 		defer xl.bp.Put(buffer)
 	case size < blockSizeV1:
 		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
-		buffer = make([]byte, size, 2*size)
+		buffer = make([]byte, size, 2*size+int64(erasure.parityBlocks+erasure.dataBlocks-1))
 	}
 
 	if len(buffer) > int(xlMeta.Erasure.BlockSize) {
@@ -969,7 +970,7 @@ func (xl xlObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuat
 	return listObjectsV2Info, err
 }
 
-// Send the successul but partial upload, however ignore
+// Send the successful but partial upload, however ignore
 // if the channel is blocked by other items.
 func (xl xlObjects) addPartialUpload(bucket, key string) {
 	select {
@@ -977,3 +978,60 @@ func (xl xlObjects) addPartialUpload(bucket, key string) {
 	default:
 	}
 }
+
+// PutObjectTag - replace or add tags to an existing object
+func (xl xlObjects) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	disks := xl.getDisks()
+
+	// Read metadata associated with the object from all disks.
+	metaArr, errs := readAllXLMetadata(ctx, disks, bucket, object)
+
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
+	if err != nil {
+		return err
+	}
+
+	for i, xlMeta := range metaArr {
+		// clean xlMeta.Meta of tag key, before updating the new tags
+		delete(xlMeta.Meta, xhttp.AmzObjectTagging)
+		// Don't update for empty tags
+		if tags != "" {
+			xlMeta.Meta[xhttp.AmzObjectTagging] = tags
+		}
+		metaArr[i].Meta = xlMeta.Meta
+	}
+
+	tempObj := mustGetUUID()
+
+	// Write unique `xl.json` for each disk.
+	if disks, err = writeUniqueXLMetadata(ctx, disks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// Atomically rename `xl.json` from tmp location to destination for each disk.
+	if _, err = renameXLMetadata(ctx, disks, minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	return nil
+}
+
+// DeleteObjectTag - delete object tags from an existing object
+func (xl xlObjects) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	return xl.PutObjectTag(ctx, bucket, object, "")
+}
+
+// GetObjectTag - get object tags from an existing object
+func (xl xlObjects) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	// GetObjectInfo will return tag value as well
+	oi, err := xl.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+
+	tags, err := tagging.FromString(oi.UserTags)
+	if err != nil {
+		return tagging.Tagging{}, err
+	}
+	return tags, nil
+}

cmd/xl-v1-object_test.go

@@ -217,12 +217,14 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
 	// for a 16 disk setup, quorum is 9. To simulate disks not found yet
 	// quorum is available, we remove disks leaving quorum disks behind.
 	xlDisks := xl.getDisks()
+	z.zones[0].xlDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		for i := range xlDisks[:7] {
 			xlDisks[i] = newNaughtyDisk(xlDisks[i], nil, errFaultyDisk)
 		}
 		return xlDisks
 	}
+	z.zones[0].xlDisksMu.Unlock()
 	err = obj.DeleteObject(context.Background(), bucket, object)
 	if err != nil {
 		t.Fatal(err)
@@ -236,11 +238,13 @@ func TestXLDeleteObjectDiskNotFound(t *testing.T) {
 
 	// Remove one more disk to 'lose' quorum, by setting it to nil.
 	xlDisks = xl.getDisks()
+	z.zones[0].xlDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		xlDisks[7] = nil
 		xlDisks[8] = nil
 		return xlDisks
 	}
+	z.zones[0].xlDisksMu.Unlock()
 	err = obj.DeleteObject(context.Background(), bucket, object)
 	// since majority of disks are not available, metaquorum is not achieved and hence errXLReadQuorum error
 	if err != toObjectErr(errXLReadQuorum, bucket, object) {
@@ -293,9 +297,11 @@ func TestGetObjectNoQuorum(t *testing.T) {
 				xlDisks[i] = newNaughtyDisk(xlDisks[i], diskErrors, errFaultyDisk)
 			}
 		}
+		z.zones[0].xlDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI {
 			return xlDisks
 		}
+		z.zones[0].xlDisksMu.Unlock()
 		// Fetch object from store.
 		err = xl.GetObject(context.Background(), bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts)
 		if err != toObjectErr(errXLReadQuorum, bucket, object) {
@@ -350,9 +356,11 @@ func TestPutObjectNoQuorum(t *testing.T) {
 				xlDisks[i] = newNaughtyDisk(xlDisks[i], diskErrors, errFaultyDisk)
 			}
 		}
+		z.zones[0].xlDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI {
 			return xlDisks
 		}
+		z.zones[0].xlDisksMu.Unlock()
 		// Upload new content to same object "object"
 		_, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
 		if err != toObjectErr(errXLWriteQuorum, bucket, object) {

cmd/xl-v1.go

@@ -192,22 +192,23 @@ func (xl xlObjects) StorageInfo(ctx context.Context) StorageInfo {
 	return getStorageInfo(xl.getDisks())
 }
 
-// GetMetrics - no op
+// GetMetrics - is not implemented and shouldn't be called.
 func (xl xlObjects) GetMetrics(ctx context.Context) (*Metrics, error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return &Metrics{}, NotImplemented{}
 }
 
-func (xl xlObjects) crawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+// CrawlAndGetDataUsage picks three random disks to crawl and get data usage
+func (xl xlObjects) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
 	var randomDisks []StorageAPI
 	for _, d := range xl.getLoadBalancedDisks() {
 		if d == nil || !d.IsOnline() {
 			continue
 		}
-		if len(randomDisks) > 3 {
+		randomDisks = append(randomDisks, d)
+		if len(randomDisks) >= 3 {
 			break
 		}
-		randomDisks = append(randomDisks, d)
 	}
 
 	var dataUsageResults = make([]DataUsageInfo, len(randomDisks))

cmd/xl-zones.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/policy"
 	"github.com/minio/minio/pkg/sync/errgroup"
+	"github.com/minio/minio/pkg/tagging"
 )
 
 type xlZones struct {
@@ -62,9 +63,7 @@ func newXLZones(endpointZones EndpointZones) (ObjectLayer, error) {
 	)
 	local := endpointZones.FirstLocal()
 	for i, ep := range endpointZones {
-		logger.Info("Formatting %v zone, %v set(s), %v drives per set.",
-			i+1, ep.SetCount, ep.DrivesPerSet)
-		formats[i], err = waitForFormatXL(local, ep.Endpoints,
+		formats[i], err = waitForFormatXL(local, ep.Endpoints, i+1,
 			ep.SetCount, ep.DrivesPerSet, deploymentID)
 		if err != nil {
 			return nil, err
@@ -213,7 +212,7 @@ func (z *xlZones) StorageInfo(ctx context.Context) StorageInfo {
 	return storageInfo
 }
 
-func (z *xlZones) crawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
+func (z *xlZones) CrawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{}) DataUsageInfo {
 	var aggDataUsageInfo = struct {
 		sync.Mutex
 		DataUsageInfo
@@ -228,7 +227,7 @@ func (z *xlZones) crawlAndGetDataUsage(ctx context.Context, endCh <-chan struct{
 			wg.Add(1)
 			go func(xl *xlObjects) {
 				defer wg.Done()
-				info := xl.crawlAndGetDataUsage(ctx, endCh)
+				info := xl.CrawlAndGetDataUsage(ctx, endCh)
 
 				aggDataUsageInfo.Lock()
 				aggDataUsageInfo.ObjectsCount += info.ObjectsCount
@@ -1376,3 +1375,63 @@ func (z *xlZones) GetMetrics(ctx context.Context) (*Metrics, error) {
 func (z *xlZones) IsReady(ctx context.Context) bool {
 	return z.zones[0].IsReady(ctx)
 }
+
+// PutObjectTag - replace or add tags to an existing object
+func (z *xlZones) PutObjectTag(ctx context.Context, bucket, object string, tags string) error {
+	if z.SingleZone() {
+		return z.zones[0].PutObjectTag(ctx, bucket, object, tags)
+	}
+	for _, zone := range z.zones {
+		err := zone.PutObjectTag(ctx, bucket, object, tags)
+		if err != nil {
+			if isErrBucketNotFound(err) {
+				continue
+			}
+			return err
+		}
+		return nil
+	}
+	return BucketNotFound{
+		Bucket: bucket,
+	}
+}
+
+// DeleteObjectTag - delete object tags from an existing object
+func (z *xlZones) DeleteObjectTag(ctx context.Context, bucket, object string) error {
+	if z.SingleZone() {
+		return z.zones[0].DeleteObjectTag(ctx, bucket, object)
+	}
+	for _, zone := range z.zones {
+		err := zone.DeleteObjectTag(ctx, bucket, object)
+		if err != nil {
+			if isErrBucketNotFound(err) {
+				continue
+			}
+			return err
+		}
+		return nil
+	}
+	return BucketNotFound{
+		Bucket: bucket,
+	}
+}
+
+// GetObjectTag - get object tags from an existing object
+func (z *xlZones) GetObjectTag(ctx context.Context, bucket, object string) (tagging.Tagging, error) {
+	if z.SingleZone() {
+		return z.zones[0].GetObjectTag(ctx, bucket, object)
+	}
+	for _, zone := range z.zones {
+		tags, err := zone.GetObjectTag(ctx, bucket, object)
+		if err != nil {
+			if isErrBucketNotFound(err) {
+				continue
+			}
+			return tags, err
+		}
+		return tags, nil
+	}
+	return tagging.Tagging{}, BucketNotFound{
+		Bucket: bucket,
+	}
+}

code_of_conduct.md

@@ -0,0 +1,80 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+In the interest of fostering an open and welcoming environment, we as
+contributors and maintainers pledge to making participation in our project and
+our community a harassment-free experience for everyone, regardless of age, body
+size, disability, ethnicity, gender identity and expression, level of experience,
+nationality, personal appearance, race, religion, or sexual identity and
+orientation.
+
+## Our Standards
+
+Examples of behavior that contributes to creating a positive environment
+include:
+
+* Using welcoming and inclusive language
+* Being respectful of differing viewpoints and experiences
+* Gracefully accepting constructive criticism
+* Focusing on what is best for the community
+* Showing empathy towards other community members
+
+Examples of unacceptable behavior by participants include:
+
+* The use of sexualized language or imagery and unwelcome sexual attention or
+  advances
+* Trolling, insulting/derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or electronic
+  address, without explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Our Responsibilities
+
+Project maintainers are responsible for clarifying the standards of acceptable
+behavior and are expected to take appropriate and fair corrective action in
+response to any instances of unacceptable behavior, in compliance with the
+licensing terms applying to the Project developments.
+
+Project maintainers have the right and responsibility to remove, edit, or
+reject comments, commits, code, wiki edits, issues, and other contributions
+that are not aligned to this Code of Conduct, or to ban temporarily or
+permanently any contributor for other behaviors that they deem inappropriate,
+threatening, offensive, or harmful. However, these actions shall respect the
+licensing terms of the Project Developments that will always supersede such
+Code of Conduct.
+
+## Scope
+
+This Code of Conduct applies both within project spaces and in public spaces
+when an individual is representing the project or its community. Examples of
+representing a project or community include using an official project e-mail
+address, posting via an official social media account, or acting as an appointed
+representative at an online or offline event. Representation of a project may be
+further defined and clarified by project maintainers.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported by contacting the project team at dev@min.io. The project team
+will review and investigate all complaints, and will respond in a way that it deems
+appropriate to the circumstances. The project team is obligated to maintain
+confidentiality with regard to the reporter of an incident.
+Further details of specific enforcement policies may be posted separately.
+
+Project maintainers who do not follow or enforce the Code of Conduct in good
+faith may face temporary or permanent repercussions as determined by other
+members of the project's leadership.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
+available at [http://contributor-covenant.org/version/1/4][version]
+
+This version includes a clarification to ensure that the code of conduct is in
+compliance with the free software licensing terms of the project.
+
+[homepage]: http://contributor-covenant.org
+[version]: http://contributor-covenant.org/version/1/4/

docs/bucket/notifications/README.md

@@ -77,7 +77,7 @@ KEY:
 notify_amqp[:name]  publish bucket notifications to AMQP endpoints
 
 ARGS:
-MINIO_NOTIFY_AMQP_ENABLE         (on|off)    enable amqp notifications, default is 'off'
+MINIO_NOTIFY_AMQP_ENABLE*        (on|off)    enable notify_amqp target, default is 'off'
 MINIO_NOTIFY_AMQP_URL*           (url)       AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672`
 MINIO_NOTIFY_AMQP_EXCHANGE       (string)    name of the AMQP exchange
 MINIO_NOTIFY_AMQP_EXCHANGE_TYPE  (string)    AMQP exchange type
@@ -211,6 +211,7 @@ KEY:
 notify_mqtt[:name]  publish bucket notifications to MQTT endpoints
 
 ARGS:
+MINIO_NOTIFY_MQTT_ENABLE*              (on|off)    enable notify_mqtt target, default is 'off'
 MINIO_NOTIFY_MQTT_BROKER*              (uri)       MQTT server endpoint e.g. `tcp://localhost:1883`
 MINIO_NOTIFY_MQTT_TOPIC*               (string)    name of the MQTT topic to publish
 MINIO_NOTIFY_MQTT_USERNAME             (string)    MQTT username
@@ -343,6 +344,7 @@ KEY:
 notify_elasticsearch[:name]  publish bucket notifications to Elasticsearch endpoints
 
 ARGS:
+MINIO_NOTIFY_ELASTICSEARCH_ENABLE*      (on|off)             enable notify_elasticsearch target, default is 'off'
 MINIO_NOTIFY_ELASTICSEARCH_URL*         (url)                Elasticsearch server's address, with optional authentication info
 MINIO_NOTIFY_ELASTICSEARCH_INDEX*       (string)             Elasticsearch index to store/update events, index is auto-created
 MINIO_NOTIFY_ELASTICSEARCH_FORMAT*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
@@ -508,7 +510,7 @@ KEY:
 notify_redis[:name]  publish bucket notifications to Redis datastores
 
 ARGS:
-MINIO_NOTIFY_REDIS_ADDRESS*     (address)            Redis server's address. For example: `localhost:6379`
+MINIO_NOTIFY_REDIS_ENABLE*      (on|off)             enable notify_redis target, default is 'off'
 MINIO_NOTIFY_REDIS_KEY*         (string)             Redis key to store/update events, key is auto-created
 MINIO_NOTIFY_REDIS_FORMAT*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
 MINIO_NOTIFY_REDIS_PASSWORD     (string)             Redis server password
@@ -619,6 +621,7 @@ KEY:
 notify_nats[:name]  publish bucket notifications to NATS endpoints
 
 ARGS:
+MINIO_NOTIFY_NATS_ENABLE*                           (on|off)    enable notify_nats target, default is 'off'
 MINIO_NOTIFY_NATS_ADDRESS*                          (address)   NATS server address e.g. '0.0.0.0:4222'
 MINIO_NOTIFY_NATS_SUBJECT*                          (string)    NATS subscription subject
 MINIO_NOTIFY_NATS_USERNAME                          (string)    NATS username
@@ -843,6 +846,7 @@ KEY:
 notify_postgres[:name]  publish bucket notifications to Postgres databases
 
 ARGS:
+MINIO_NOTIFY_POSTGRES_ENABLE*             (on|off)             enable notify_postgres target, default is 'off'
 MINIO_NOTIFY_POSTGRES_CONNECTION_STRING*  (string)             Postgres server connection-string
 MINIO_NOTIFY_POSTGRES_TABLE*              (string)             DB table name to store/update events, table is auto-created
 MINIO_NOTIFY_POSTGRES_FORMAT*             (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
@@ -959,6 +963,7 @@ KEY:
 notify_mysql[:name]  publish bucket notifications to MySQL databases
 
 ARGS:
+MINIO_NOTIFY_MYSQL_ENABLE*      (on|off)             enable notify_mysql target, default is 'off'
 MINIO_NOTIFY_MYSQL_DSN_STRING*  (string)             MySQL data-source-name connection string
 MINIO_NOTIFY_MYSQL_TABLE*       (string)             DB table name to store/update events, table is auto-created
 MINIO_NOTIFY_MYSQL_FORMAT*      (namespace*|access)  'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'
@@ -1071,6 +1076,7 @@ KEY:
 notify_kafka[:name]  publish bucket notifications to Kafka endpoints
 
 ARGS:
+MINIO_NOTIFY_KAFKA_ENABLE*          (on|off)    enable notify_kafka target, default is 'off'
 MINIO_NOTIFY_KAFKA_BROKERS*         (csv)       comma separated list of Kafka broker addresses
 MINIO_NOTIFY_KAFKA_TOPIC            (string)    Kafka topic used for bucket notifications
 MINIO_NOTIFY_KAFKA_SASL_USERNAME    (string)    username for SASL/PLAIN or SASL/SCRAM authentication
@@ -1213,6 +1219,7 @@ KEY:
 notify_webhook[:name]  publish bucket notifications to webhook endpoints
 
 ARGS:
+MINIO_NOTIFY_WEBHOOK_ENABLE*      (on|off)    enable notify_webhook target, default is 'off'
 MINIO_NOTIFY_WEBHOOK_ENDPOINT*    (url)       webhook server endpoint e.g. http://localhost:8080/minio/events
 MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN   (string)    opaque string or JWT authorization token
 MINIO_NOTIFY_WEBHOOK_QUEUE_DIR    (path)      staging dir for undelivered messages e.g. '/home/events'
@@ -1318,6 +1325,7 @@ KEY:
 notify_nsq[:name]  publish bucket notifications to NSQ endpoints
 
 ARGS:
+MINIO_NOTIFY_NSQ_ENABLE*          (on|off)    enable notify_nsq target, default is 'off'
 MINIO_NOTIFY_NSQ_NSQD_ADDRESS*    (address)   NSQ server address e.g. '127.0.0.1:4150'
 MINIO_NOTIFY_NSQ_TOPIC*           (string)    NSQ topic
 MINIO_NOTIFY_NSQ_TLS              (on|off)    set to 'on' to enable TLS

docs/compression/README.md

@@ -19,6 +19,7 @@ Compression can be enabled by updating the `compress` config settings for MinIO
 ```
 $ mc admin config get myminio compression
 compression extensions=".txt,.log,.csv,.json,.tar,.xml,.bin" mime_types="text/*,application/json,application/xml"```
+```
 
 Default config includes most common highly compressible content extensions and mime-types.
 

docs/deployment/kernel-tuning/sysctl.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-cat > sysctl.conf <EOF
+cat > sysctl.conf <<EOF
 # maximum number of open files/file descriptors
 fs.file-max = 4194303
 

docs/minio-limits.md

@@ -51,7 +51,6 @@ We found the following APIs to be redundant or less useful outside of AWS S3. If
 - ObjectACL (Use [bucket policies](https://docs.min.io/docs/minio-client-complete-guide#policy) instead)
 - ObjectTorrent
 - ObjectVersions
-- ObjectTagging
 
 ### Object name restrictions on MinIO
 Object names that contain characters `^*|\/&";` are unsupported on Windows and other file systems which do not support filenames with these characters. Note that this list is not exhaustive, and depends on the maintainers of the filesystem itself.

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # 9001 through 9004.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+    image: minio/minio:RELEASE.2020-01-16T22-40-29Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+        image: minio/minio:RELEASE.2020-01-16T22-40-29Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -22,7 +22,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+        image: minio/minio:RELEASE.2020-01-16T22-40-29Z
         args:
         - server
         - http://minio-{0...3}.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -24,7 +24,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+        image: minio/minio:RELEASE.2020-01-16T22-40-29Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -32,7 +32,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2020-01-16T03-05-44Z
+        image: minio/minio:RELEASE.2020-01-16T22-40-29Z
         args:
         - server
         - /data

docs/retention/README.md

@@ -5,6 +5,9 @@ MinIO server allows selectively specify WORM for specific objects or configuring
 Object locking requires locking to be enabled on a bucket at the time of bucket creation. In addition, a default retention period and retention mode can be configured on a bucket to be
 applied to objects created in that bucket.
 
+Independently of retention, an object can also be under legal hold. This effectively disallows
+all deletes and overwrites of an object under legal hold until the hold is lifted.
+
 ## Get Started
 
 ### 1. Prerequisites
@@ -29,10 +32,22 @@ aws s3api put-object --bucket testbucket --key lockme --object-lock-mode GOVERNA
 See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on
 object locking and permissions required for object retention and governance bypass overrides.
 
+### Set legal hold on an object
+
+PutObject API allows setting legal hold using `x-amz-object-lock-legal-hold` header.
+
+```sh
+aws s3api put-object --bucket testbucket --key legalhold --object-lock-legal-hold-status ON --body /etc/issue
+```
+
+See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html for AWS S3 spec on
+object locking and permissions required for specifying legal hold.
+
 ### 3. Note
 
 - When global WORM is enabled by `MINIO_WORM` environment variable or `worm` field in configuration file supersedes bucket level WORM and `PUT object lock configuration` REST API is disabled.
 - In global WORM mode objects can never be overwritten
+- If an object is under legal hold, it cannot be overwritten unless the legal hold is explicitly removed.
 - In `Compliance` mode, objects cannot be overwritten or deleted by anyone until retention period
 is expired. If user has requisite governance bypass permissions, an object's retention date can
 be extended in `Compliance` mode.

docs/zh_CN/distributed/README.md

@@ -72,10 +72,13 @@ minio.exe server http://192.168.1.11/C:/data http://192.168.1.12/C:/data ^
                   http://192.168.1.17/C:/data http://192.168.1.18/C:/data
 ```
 
+
 ![分布式Minio,8节点，每个节点一块盘](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_8.jpg?raw=true)
 
 示例2: 启动分布式Minio实例，4节点，每节点4块盘，需要在4个节点上都运行下面的命令。
 
+
+
 #### GNU/Linux 和 macOS
 
 ```shell
@@ -108,6 +111,29 @@ minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.11/C:/data2 ^
 
 ![分布式Minio,4节点，每节点4块盘](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_16.jpg?raw=true)
 
+#### 扩展现有的分布式集群
+例如我们是通过区的方式启动MinIO集群，命令行如下：
+
+```shell
+export MINIO_ACCESS_KEY=<ACCESS_KEY>
+export MINIO_SECRET_KEY=<SECRET_KEY>
+minio server http://host{1...32}/export{1...32}
+```
+
+MinIO支持通过命令，指定新的集群来扩展现有集群（纠删码模式），命令行如下：
+
+```shell
+export MINIO_ACCESS_KEY=<ACCESS_KEY>
+export MINIO_SECRET_KEY=<SECRET_KEY>
+minio server http://host{1...32}/export{1...32} http://host{33...64}/export{1...32}
+```
+
+现在整个集群就扩展了1024个磁盘，总磁盘变为2048个，新的对象上传请求会自动分配到最少使用的集群上。通过以上扩展策略，您就可以按需扩展您的集群。重新配置后重启集群，会立即在集群中生效，并对现有集群无影响。如上命令中，我们可以把原来的集群看做一个区，新增集群看做另一个区，新对象按每个区域中的可用空间比例放置在区域中。在每个区域内，基于确定性哈希算法确定位置。
+
+> __说明:__ __您添加的每个区域必须具有与原始区域相同的磁盘数量（纠删码集）大小，以便维持相同的数据冗余SLA。__ 
+> 例如，第一个区有8个磁盘，您可以将集群扩展为16个、32个或1024个磁盘的区域，您只需确保部署的SLA是原始区域的倍数即可。
+
+
 ## 3. 验证
 
 验证是否部署成功，使用浏览器访问Minio服务或者使用 [`mc`](https://docs.min.io/cn/minio-client-quickstart-guide)。多个节点的存储容量和就是分布式Minio的存储容量。

go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/gopherjs/gopherjs v0.0.0-20190328170749-bb2674552d8f // indirect
 	github.com/gorilla/handlers v1.4.0
 	github.com/gorilla/mux v1.7.0
-	github.com/gorilla/rpc v1.2.0+incompatible
+	github.com/gorilla/rpc v1.2.0
 	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
@@ -67,7 +67,7 @@ require (
 	github.com/minio/hdfs/v3 v3.0.1
 	github.com/minio/highwayhash v1.0.0
 	github.com/minio/lsync v1.0.1
-	github.com/minio/minio-go/v6 v6.0.44
+	github.com/minio/minio-go/v6 v6.0.45-0.20200117140906-66cf57d21ba4
 	github.com/minio/parquet-go v0.0.0-20191231003236-20b3c07bcd2c
 	github.com/minio/sha256-simd v0.1.1
 	github.com/minio/sio v0.2.0
@@ -120,6 +120,3 @@ require (
 	gopkg.in/olivere/elastic.v5 v5.0.80
 	gopkg.in/yaml.v2 v2.2.4
 )
-
-// Added for go1.13 migration https://github.com/golang/go/issues/32805
-replace github.com/gorilla/rpc v1.2.0+incompatible => github.com/gorilla/rpc v1.2.0

go.sum

@@ -219,15 +219,9 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/compress v1.8.3 h1:CkLseiEYMM/fRb0RIg9mXB+Iwgmle+U9KGFu+JCO4Ec=
-github.com/klauspost/compress v1.8.3/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.9.4/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.9.7 h1:hYW1gP94JUmAhBtJ+LNz5My+gBobDxPR1iVuKug26aA=
 github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
-github.com/klauspost/cpuid v0.0.0-20160106104451-349c67577817/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
-github.com/klauspost/cpuid v1.2.1 h1:vJi+O/nMdFt0vqm8NZBI6wzALWdA2X+egi0ogNyrC/w=
-github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.2 h1:1xAgYebNnsb9LKCdLOvFWtAxGU/33mjJtyOVbmUa0Us=
 github.com/klauspost/cpuid v1.2.2/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/pgzip v1.2.1 h1:oIPZROsWuPHpOdMVWLuJZXwgjhrW8r1yEX8UqMyeNHM=
@@ -278,6 +272,8 @@ github.com/minio/lsync v1.0.1 h1:AVvILxA976xc27hstd1oR+X9PQG0sPSom1MNb1ImfUs=
 github.com/minio/lsync v1.0.1/go.mod h1:tCFzfo0dlvdGl70IT4IAK/5Wtgb0/BrTmo/jE8pArKA=
 github.com/minio/minio-go/v6 v6.0.44 h1:CVwVXw+uCOcyMi7GvcOhxE8WgV+Xj8Vkf2jItDf/EGI=
 github.com/minio/minio-go/v6 v6.0.44/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
+github.com/minio/minio-go/v6 v6.0.45-0.20200117140906-66cf57d21ba4 h1:zjzoKgiRJm7k0J+IXz36u+ltbbU/iRTBlbm5/vZFt1w=
+github.com/minio/minio-go/v6 v6.0.45-0.20200117140906-66cf57d21ba4/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/parquet-go v0.0.0-20191231003236-20b3c07bcd2c h1:PXr/55MEoFh0G6NeV0plWrdpTDr5xaDTxP3SpavSdGc=
 github.com/minio/parquet-go v0.0.0-20191231003236-20b3c07bcd2c/go.mod h1:sl82d+TnCE7qeaNJazHdNoG9Gpyl9SZYfleDAQWrsls=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
@@ -344,8 +340,6 @@ github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pkg/profile v1.3.0 h1:OQIvuDgm00gWVWGTf4m4mCt6W1/0YqU7Ntg0mySWgaI=
-github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=

hound.yml

@@ -1,3 +0,0 @@
-jshint:
-    enabled: false
-

mint/run/core/aws-sdk-go/quick-tests.go

@@ -28,10 +28,12 @@ import (
 	"math/rand"
 	"net/http"
 	"os"
+	"reflect"
 	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
@@ -132,6 +134,50 @@ func randString(n int, src rand.Source, prefix string) string {
 	return prefix + string(b[0:30-len(prefix)])
 }
 
+func isObjectTaggingImplemented(s3Client *s3.S3) bool {
+	bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-")
+	object := randString(60, rand.NewSource(time.Now().UnixNano()), "")
+	startTime := time.Now()
+	function := "isObjectTaggingImplemented"
+	args := map[string]interface{}{
+		"bucketName": bucket,
+		"objectName": object,
+	}
+	defer cleanup(s3Client, bucket, object, function, args, startTime, true)
+
+	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal()
+		return false
+	}
+
+	_, err = s3Client.PutObject(&s3.PutObjectInput{
+		Body:   aws.ReadSeekCloser(strings.NewReader("testfile")),
+		Bucket: aws.String(bucket),
+		Key:    aws.String(object),
+	})
+
+	if err != nil {
+		failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal()
+		return false
+	}
+
+	_, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(object),
+	})
+	if err != nil {
+		if awsErr, ok := err.(awserr.Error); ok {
+			if awsErr.Code() == "NotImplemented" {
+				return false
+			}
+		}
+	}
+	return true
+}
+
 func cleanup(s3Client *s3.S3, bucket string, object string, function string,
 	args map[string]interface{}, startTime time.Time, deleteBucket bool) {
 
@@ -474,6 +520,95 @@ func testSelectObject(s3Client *s3.S3) {
 	successLogger(function, args, startTime).Info()
 }
 
+func testObjectTagging(s3Client *s3.S3) {
+	startTime := time.Now()
+	function := "testObjectTagging"
+	bucket := randString(60, rand.NewSource(time.Now().UnixNano()), "aws-sdk-go-test-")
+	object := randString(60, rand.NewSource(time.Now().UnixNano()), "")
+	args := map[string]interface{}{
+		"bucketName": bucket,
+		"objectName": object,
+	}
+
+	_, err := s3Client.CreateBucket(&s3.CreateBucketInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		failureLog(function, args, startTime, "", "AWS SDK Go CreateBucket Failed", err).Fatal()
+		return
+	}
+	defer cleanup(s3Client, bucket, object, function, args, startTime, true)
+
+	taginput := "Tag1=Value1"
+	tagInputSet := []*s3.Tag{
+		{
+			Key:   aws.String("Tag1"),
+			Value: aws.String("Value1"),
+		},
+	}
+	_, err = s3Client.PutObject(&s3.PutObjectInput{
+		Body:    aws.ReadSeekCloser(strings.NewReader("testfile")),
+		Bucket:  aws.String(bucket),
+		Key:     aws.String(object),
+		Tagging: &taginput,
+	})
+
+	if err != nil {
+		failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUT expected to success but got %v", err), err).Fatal()
+		return
+	}
+
+	tagop, err := s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(object),
+	})
+	if err != nil {
+		if awsErr, ok := err.(awserr.Error); ok {
+			failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal()
+			return
+		}
+	}
+	if !reflect.DeepEqual(tagop.TagSet, tagInputSet) {
+		failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObject Tag input did not match with GetObjectTagging output %v", nil), nil).Fatal()
+		return
+	}
+
+	taginputSet1 := []*s3.Tag{
+		{
+			Key:   aws.String("Key4"),
+			Value: aws.String("Value4"),
+		},
+	}
+	_, err = s3Client.PutObjectTagging(&s3.PutObjectTaggingInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(object),
+		Tagging: &s3.Tagging{
+			TagSet: taginputSet1,
+		},
+	})
+	if err != nil {
+		if awsErr, ok := err.(awserr.Error); ok {
+			failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal()
+			return
+		}
+	}
+
+	tagop, err = s3Client.GetObjectTagging(&s3.GetObjectTaggingInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(object),
+	})
+	if err != nil {
+		if awsErr, ok := err.(awserr.Error); ok {
+			failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging expected to success but got %v", awsErr.Code()), err).Fatal()
+			return
+		}
+	}
+	if !reflect.DeepEqual(tagop.TagSet, taginputSet1) {
+		failureLog(function, args, startTime, "", fmt.Sprintf("AWS SDK Go PUTObjectTagging input did not match with GetObjectTagging output %v", nil), nil).Fatal()
+		return
+	}
+}
+
 func main() {
 	endpoint := os.Getenv("SERVER_ENDPOINT")
 	accessKey := os.Getenv("ACCESS_KEY")
@@ -508,4 +643,7 @@ func main() {
 	testPresignedPutInvalidHash(s3Client)
 	testListObjects(s3Client)
 	testSelectObject(s3Client)
+	if isObjectTaggingImplemented(s3Client) {
+		testObjectTagging(s3Client)
+	}
 }

mint/run/core/awscli/test.sh

@@ -69,6 +69,24 @@ function make_bucket() {
     return $rv
 }
 
+function make_bucket_with_lock() {
+    # Make bucket
+    bucket_name="awscli-mint-test-bucket-$RANDOM"
+    function="${AWS} s3api create-bucket --bucket ${bucket_name} --object-lock-enabled-for-bucket"
+
+    # execute the test
+    out=$($function 2>&1)
+    rv=$?
+
+    # if command is successful print bucket_name or print error
+    if [ $rv -eq 0 ]; then
+        echo "${bucket_name}"
+    else
+        echo "${out}"
+    fi
+
+    return $rv
+}
 function delete_bucket() {
     # Delete bucket
     function="${AWS} s3 rb s3://${1} --force"
@@ -1430,6 +1448,98 @@ function test_worm_bucket() {
     return $rv
 }
 
+# Tests creating and deleting an object with legal hold.
+function test_legal_hold() {
+    # log start time
+    start_time=$(get_time)
+
+    function="make_bucket_with_lock"
+    bucket_name=$(make_bucket_with_lock)
+    rv=$?
+    # if make bucket succeeds upload a file
+    if [ $rv -eq 0 ]; then
+        function="${AWS} s3api put-object --body ${MINT_DATA_DIR}/datafile-1-kB --bucket ${bucket_name} --key datafile-1-kB --object-lock-legal-hold-status ON"
+        out=$($function 2>&1)
+        errcnt=$(echo "$out" | sed -n '/Bucket is missing ObjectLockConfiguration/p' | wc -l)
+        # skip test for gateways
+        if [ "$errcnt" -eq 1 ]; then
+            return 0
+        fi
+        rv=$?
+    else
+        # if make bucket fails, $bucket_name has the error output
+        out="${bucket_name}"
+    fi
+
+    # if upload succeeds download the file
+    if [ $rv -eq 0 ]; then
+        function="${AWS} s3api head-object --bucket ${bucket_name} --key datafile-1-kB"
+        # save the ref to function being tested, so it can be logged
+        test_function=${function}
+        out=$($function 2>&1)
+        lhold=$(echo "$out" | jq -r .ObjectLockLegalHoldStatus)
+        rv=$?
+    fi
+
+    # if head-object succeeds, verify metadata has legal hold status
+    if [ $rv -eq 0 ]; then
+       if [ "${lhold}" == "" ]; then
+            rv=1
+            out="Legal hold was not applied"
+        fi
+        if [ "${lhold}" == "OFF" ]; then
+            rv=1
+            out="Legal hold was not applied"
+        fi
+    fi
+    if [ $rv -eq 0 ]; then
+        function="${AWS} s3api put-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB --legal-hold Status=OFF"
+        out=$($function 2>&1)
+        rv=$?
+    else
+        # if make bucket fails, $bucket_name has the error output
+        out="${bucket_name}"
+    fi
+    # if upload succeeds download the file
+    if [ $rv -eq 0 ]; then
+        function="${AWS} s3api get-object-legal-hold --bucket ${bucket_name} --key datafile-1-kB"
+        # save the ref to function being tested, so it can be logged
+        test_function=${function}
+        out=$($function 2>&1)
+        lhold=$(echo "$out" | jq -r .LegalHold.Status)
+        rv=$?
+    fi
+
+    # if head-object succeeds, verify metadata has legal hold status
+    if [ $rv -eq 0 ]; then
+       if [ "${lhold}" == "" ]; then
+            rv=1
+            out="Legal hold was not applied"
+        fi
+        if [ "${lhold}" == "ON" ]; then
+            rv=1
+            out="Legal hold status not turned off"
+        fi
+    fi
+     # Attempt a delete on prefix shouldn't delete the directory since we have an object inside it.
+    if [ $rv -eq 0 ]; then
+        function="${AWS} s3api delete-object --bucket ${bucket_name} --key datafile-1-kB"
+        # save the ref to function being tested, so it can be logged
+        test_function=${function}
+        out=$($function 2>&1)
+        rv=$?
+    fi
+    if [ $rv -eq 0 ]; then
+        log_success "$(get_duration "$start_time")" "${test_function}"
+    else
+        # clean up and log error
+        ${AWS} s3 rb s3://"${bucket_name}" --force > /dev/null 2>&1
+        log_failure "$(get_duration "$start_time")" "${function}" "${out}"
+    fi
+
+    return $rv
+}
+
 # main handler for all the tests.
 main() {
     # Success tests
@@ -1455,7 +1565,9 @@ main() {
     test_list_objects_error && \
     test_put_object_error && \
     test_serverside_encryption_error && \
-    test_worm_bucket
+    test_worm_bucket && \
+    test_legal_hold 
+
     return $?
 }
 

pkg/event/name.go

@@ -30,6 +30,7 @@ const (
 	ObjectAccessedAll Name = 1 + iota
 	ObjectAccessedGet
 	ObjectAccessedGetRetention
+	ObjectAccessedGetLegalHold
 	ObjectAccessedHead
 	ObjectCreatedAll
 	ObjectCreatedCompleteMultipartUpload
@@ -37,6 +38,7 @@ const (
 	ObjectCreatedPost
 	ObjectCreatedPut
 	ObjectCreatedPutRetention
+	ObjectCreatedPutLegalHold
 	ObjectRemovedAll
 	ObjectRemovedDelete
 )
@@ -45,9 +47,9 @@ const (
 func (name Name) Expand() []Name {
 	switch name {
 	case ObjectAccessedAll:
-		return []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention}
+		return []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}
 	case ObjectCreatedAll:
-		return []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention}
+		return []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention, ObjectCreatedPutLegalHold}
 	case ObjectRemovedAll:
 		return []Name{ObjectRemovedDelete}
 	default:
@@ -64,6 +66,8 @@ func (name Name) String() string {
 		return "s3:ObjectAccessed:Get"
 	case ObjectAccessedGetRetention:
 		return "s3:ObjectAccessed:GetRetention"
+	case ObjectAccessedGetLegalHold:
+		return "s3:ObjectAccessed:GetLegalHold"
 	case ObjectAccessedHead:
 		return "s3:ObjectAccessed:Head"
 	case ObjectCreatedAll:
@@ -78,6 +82,8 @@ func (name Name) String() string {
 		return "s3:ObjectCreated:Put"
 	case ObjectCreatedPutRetention:
 		return "s3:ObjectCreated:PutRetention"
+	case ObjectCreatedPutLegalHold:
+		return "s3:ObjectCreated:PutLegalHold"
 	case ObjectRemovedAll:
 		return "s3:ObjectRemoved:*"
 	case ObjectRemovedDelete:
@@ -138,6 +144,8 @@ func ParseName(s string) (Name, error) {
 		return ObjectAccessedGet, nil
 	case "s3:ObjectAccessed:GetRetention":
 		return ObjectAccessedGetRetention, nil
+	case "s3:ObjectAccessed:GetLegalHold":
+		return ObjectAccessedGetLegalHold, nil
 	case "s3:ObjectAccessed:Head":
 		return ObjectAccessedHead, nil
 	case "s3:ObjectCreated:*":
@@ -152,6 +160,8 @@ func ParseName(s string) (Name, error) {
 		return ObjectCreatedPut, nil
 	case "s3:ObjectCreated:PutRetention":
 		return ObjectCreatedPutRetention, nil
+	case "s3:ObjectCreated:PutLegalHold":
+		return ObjectCreatedPutLegalHold, nil
 	case "s3:ObjectRemoved:*":
 		return ObjectRemovedAll, nil
 	case "s3:ObjectRemoved:Delete":

pkg/event/name_test.go

@@ -28,8 +28,8 @@ func TestNameExpand(t *testing.T) {
 		name           Name
 		expectedResult []Name
 	}{
-		{ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention}},
-		{ObjectCreatedAll, []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention}},
+		{ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}},
+		{ObjectCreatedAll, []Name{ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, ObjectCreatedPutRetention, ObjectCreatedPutLegalHold}},
 		{ObjectRemovedAll, []Name{ObjectRemovedDelete}},
 		{ObjectAccessedHead, []Name{ObjectAccessedHead}},
 	}
@@ -60,6 +60,11 @@ func TestNameString(t *testing.T) {
 		{ObjectCreatedPut, "s3:ObjectCreated:Put"},
 		{ObjectRemovedAll, "s3:ObjectRemoved:*"},
 		{ObjectRemovedDelete, "s3:ObjectRemoved:Delete"},
+		{ObjectCreatedPutRetention, "s3:ObjectCreated:PutRetention"},
+		{ObjectCreatedPutLegalHold, "s3:ObjectCreated:PutLegalHold"},
+		{ObjectAccessedGetRetention, "s3:ObjectAccessed:GetRetention"},
+		{ObjectAccessedGetLegalHold, "s3:ObjectAccessed:GetLegalHold"},
+
 		{blankName, ""},
 	}
 

pkg/event/rulesmap_test.go

@@ -153,12 +153,12 @@ func TestRulesMapMatch(t *testing.T) {
 
 func TestNewRulesMap(t *testing.T) {
 	rulesMapCase1 := make(RulesMap)
-	rulesMapCase1.add([]Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention},
+	rulesMapCase1.add([]Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold},
 		"*", TargetID{"1", "webhook"})
 
 	rulesMapCase2 := make(RulesMap)
 	rulesMapCase2.add([]Name{ObjectAccessedGet, ObjectAccessedHead,
-		ObjectCreatedPut, ObjectAccessedGetRetention}, "*", TargetID{"1", "webhook"})
+		ObjectCreatedPut, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold}, "*", TargetID{"1", "webhook"})
 
 	rulesMapCase3 := make(RulesMap)
 	rulesMapCase3.add([]Name{ObjectRemovedDelete}, "2010*.jpg", TargetID{"1", "webhook"})

pkg/event/target/webhook.go

@@ -142,15 +142,14 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 
 	resp, err := target.httpClient.Do(req)
 	if err != nil {
+		target.Close()
 		return err
 	}
-
 	defer resp.Body.Close()
 	io.Copy(ioutil.Discard, resp.Body)
 
 	if resp.StatusCode < 200 || resp.StatusCode > 299 {
-		// close any idle connections upon any error.
-		target.httpClient.CloseIdleConnections()
+		target.Close()
 		return fmt.Errorf("sending event failed with %v", resp.Status)
 	}
 

pkg/iam/policy/action.go

@@ -102,12 +102,27 @@ const (
 	// GetObjectRetentionAction - GetObjectRetention, GetObject, HeadObject Rest API action.
 	GetObjectRetentionAction = "s3:GetObjectRetention"
 
+	// GetObjectLegalHoldAction - GetObjectLegalHold, GetObject Rest API action.
+	GetObjectLegalHoldAction = "s3:GetObjectLegalHold"
+
+	// PutObjectLegalHoldAction - PutObjectLegalHold, PutObject Rest API action.
+	PutObjectLegalHoldAction = "s3:PutObjectLegalHold"
+
 	// GetBucketObjectLockConfigurationAction - GetBucketObjectLockConfiguration Rest API action
 	GetBucketObjectLockConfigurationAction = "s3:GetBucketObjectLockConfiguration"
 
 	// PutBucketObjectLockConfigurationAction - PutBucketObjectLockConfiguration Rest API action
 	PutBucketObjectLockConfigurationAction = "s3:PutBucketObjectLockConfiguration"
 
+	// GetObjectTaggingAction - Get Object Tags API action
+	GetObjectTaggingAction = "s3:GetObjectTagging"
+
+	// PutObjectTaggingAction - Put Object Tags API action
+	PutObjectTaggingAction = "s3:PutObjectTagging"
+
+	// DeleteObjectTaggingAction - Delete Object Tags API action
+	DeleteObjectTaggingAction = "s3:DeleteObjectTagging"
+
 	// AllActions - all API actions
 	AllActions = "s3:*"
 )
@@ -137,10 +152,15 @@ var supportedActions = map[Action]struct{}{
 	PutBucketLifecycleAction:               {},
 	PutObjectRetentionAction:               {},
 	GetObjectRetentionAction:               {},
+	GetObjectLegalHoldAction:               {},
+	PutObjectLegalHoldAction:               {},
 	PutBucketObjectLockConfigurationAction: {},
 	GetBucketObjectLockConfigurationAction: {},
 	BypassGovernanceModeAction:             {},
 	BypassGovernanceRetentionAction:        {},
+	GetObjectTaggingAction:                 {},
+	PutObjectTaggingAction:                 {},
+	DeleteObjectTaggingAction:              {},
 }
 
 // isObjectAction - returns whether action is object type or not.
@@ -154,6 +174,10 @@ func (action Action) isObjectAction() bool {
 		return true
 	case PutObjectRetentionAction, GetObjectRetentionAction:
 		return true
+	case PutObjectLegalHoldAction, GetObjectLegalHoldAction:
+		return true
+	case GetObjectTaggingAction, PutObjectTaggingAction, DeleteObjectTaggingAction:
+		return true
 	}
 
 	return false
@@ -267,8 +291,13 @@ var actionConditionKeyMap = map[Action]condition.KeySet{
 		}, condition.CommonKeys...)...),
 	PutObjectRetentionAction:               condition.NewKeySet(condition.CommonKeys...),
 	GetObjectRetentionAction:               condition.NewKeySet(condition.CommonKeys...),
+	PutObjectLegalHoldAction:               condition.NewKeySet(condition.CommonKeys...),
+	GetObjectLegalHoldAction:               condition.NewKeySet(condition.CommonKeys...),
 	BypassGovernanceModeAction:             condition.NewKeySet(condition.CommonKeys...),
 	BypassGovernanceRetentionAction:        condition.NewKeySet(condition.CommonKeys...),
 	GetBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...),
 	PutBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...),
+	PutObjectTaggingAction:                 condition.NewKeySet(condition.CommonKeys...),
+	GetObjectTaggingAction:                 condition.NewKeySet(condition.CommonKeys...),
+	DeleteObjectTaggingAction:              condition.NewKeySet(condition.CommonKeys...),
 }

pkg/madmin/api.go

@@ -270,6 +270,9 @@ func (adm AdminClient) do(req *http.Request) (*http.Response, error) {
 	for {
 		resp, err = adm.httpClient.Do(req)
 		if err != nil {
+			// Close idle connections upon error.
+			adm.httpClient.CloseIdleConnections()
+
 			// Handle this specifically for now until future Golang
 			// versions fix this issue properly.
 			urlErr, ok := err.(*url.Error)

pkg/madmin/profiling-commands.go

@@ -33,11 +33,12 @@ type ProfilerType string
 
 // Different supported profiler types.
 const (
-	ProfilerCPU   ProfilerType = "cpu"   // represents CPU profiler type
-	ProfilerMEM                = "mem"   // represents MEM profiler type
-	ProfilerBlock              = "block" // represents Block profiler type
-	ProfilerMutex              = "mutex" // represents Mutex profiler type
-	ProfilerTrace              = "trace" // represents Trace profiler type
+	ProfilerCPU     ProfilerType = "cpu"     // represents CPU profiler type
+	ProfilerMEM     ProfilerType = "mem"     // represents MEM profiler type
+	ProfilerBlock   ProfilerType = "block"   // represents Block profiler type
+	ProfilerMutex   ProfilerType = "mutex"   // represents Mutex profiler type
+	ProfilerTrace   ProfilerType = "trace"   // represents Trace profiler type
+	ProfilerThreads ProfilerType = "threads" // represents ThreadCreate profiler type
 )
 
 // StartProfilingResult holds the result of starting

pkg/net/url.go

@@ -20,6 +20,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
+	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
@@ -100,6 +102,7 @@ func (u URL) DialHTTP() error {
 	if err != nil {
 		return err
 	}
+	io.Copy(ioutil.Discard, resp.Body)
 	resp.Body.Close()
 	return nil
 }

pkg/objectlock/objectlock.go

@@ -0,0 +1,521 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package objectlock
+
+import (
+	"context"
+	"encoding/xml"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/beevik/ntp"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Mode - object retention mode.
+type Mode string
+
+const (
+	// Governance - governance mode.
+	Governance Mode = "GOVERNANCE"
+
+	// Compliance - compliance mode.
+	Compliance Mode = "COMPLIANCE"
+
+	// Invalid - invalid retention mode.
+	Invalid Mode = ""
+)
+
+func parseMode(modeStr string) (mode Mode) {
+	switch strings.ToUpper(modeStr) {
+	case "GOVERNANCE":
+		mode = Governance
+	case "COMPLIANCE":
+		mode = Compliance
+	default:
+		mode = Invalid
+	}
+	return mode
+}
+
+// LegalHoldStatus - object legal hold status.
+type LegalHoldStatus string
+
+const (
+	// ON -legal hold is on.
+	ON LegalHoldStatus = "ON"
+
+	// OFF -legal hold is off.
+	OFF LegalHoldStatus = "OFF"
+)
+
+func parseLegalHoldStatus(holdStr string) LegalHoldStatus {
+	switch strings.ToUpper(holdStr) {
+	case "ON":
+		return ON
+	case "OFF":
+		return OFF
+	}
+	return LegalHoldStatus("")
+}
+
+var (
+	// ErrMalformedBucketObjectConfig -indicates that the bucket object lock config is malformed
+	ErrMalformedBucketObjectConfig = errors.New("invalid bucket object lock config")
+	// ErrInvalidRetentionDate - indicates that retention date needs to be in ISO 8601 format
+	ErrInvalidRetentionDate = errors.New("date must be provided in ISO 8601 format")
+	// ErrPastObjectLockRetainDate - indicates that retention date must be in the future
+	ErrPastObjectLockRetainDate = errors.New("the retain until date must be in the future")
+	// ErrUnknownWORMModeDirective - indicates that the retention mode is invalid
+	ErrUnknownWORMModeDirective = errors.New("unknown WORM mode directive")
+	// ErrObjectLockMissingContentMD5 - indicates missing Content-MD5 header for put object requests with locking
+	ErrObjectLockMissingContentMD5 = errors.New("content-MD5 HTTP header is required for Put Object requests with Object Lock parameters")
+	// ErrObjectLockInvalidHeaders indicates that object lock headers are missing
+	ErrObjectLockInvalidHeaders = errors.New("x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied")
+	// ErrMalformedXML - generic error indicating malformed XML
+	ErrMalformedXML = errors.New("the XML you provided was not well-formed or did not validate against our published schema")
+)
+
+const (
+	ntpServerEnv = "MINIO_NTP_SERVER"
+)
+
+var (
+	ntpServer = env.Get(ntpServerEnv, "")
+)
+
+// UTCNowNTP - is similar in functionality to UTCNow()
+// but only used when we do not wish to rely on system
+// time.
+func UTCNowNTP() (time.Time, error) {
+	// ntp server is disabled
+	if ntpServer == "" {
+		return time.Now().UTC(), nil
+	}
+	return ntp.Time(ntpServer)
+}
+
+// Retention - bucket level retention configuration.
+type Retention struct {
+	Mode     Mode
+	Validity time.Duration
+}
+
+// IsEmpty - returns whether retention is empty or not.
+func (r Retention) IsEmpty() bool {
+	return r.Mode == "" || r.Validity == 0
+}
+
+// Retain - check whether given date is retainable by validity time.
+func (r Retention) Retain(created time.Time) bool {
+	t, err := UTCNowNTP()
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		// Retain
+		return true
+	}
+	return created.Add(r.Validity).After(t)
+}
+
+// BucketObjectLockConfig - map of bucket and retention configuration.
+type BucketObjectLockConfig struct {
+	sync.RWMutex
+	retentionMap map[string]Retention
+}
+
+// Set - set retention configuration.
+func (config *BucketObjectLockConfig) Set(bucketName string, retention Retention) {
+	config.Lock()
+	config.retentionMap[bucketName] = retention
+	config.Unlock()
+}
+
+// Get - Get retention configuration.
+func (config *BucketObjectLockConfig) Get(bucketName string) (r Retention, ok bool) {
+	config.RLock()
+	defer config.RUnlock()
+	r, ok = config.retentionMap[bucketName]
+	return r, ok
+}
+
+// Remove - removes retention configuration.
+func (config *BucketObjectLockConfig) Remove(bucketName string) {
+	config.Lock()
+	delete(config.retentionMap, bucketName)
+	config.Unlock()
+}
+
+// NewBucketObjectLockConfig returns initialized BucketObjectLockConfig
+func NewBucketObjectLockConfig() *BucketObjectLockConfig {
+	return &BucketObjectLockConfig{
+		retentionMap: map[string]Retention{},
+	}
+}
+
+// DefaultRetention - default retention configuration.
+type DefaultRetention struct {
+	XMLName xml.Name `xml:"DefaultRetention"`
+	Mode    Mode     `xml:"Mode"`
+	Days    *uint64  `xml:"Days"`
+	Years   *uint64  `xml:"Years"`
+}
+
+// Maximum support retention days and years supported by AWS S3.
+const (
+	// This tested by using `mc lock` command
+	maximumRetentionDays  = 36500
+	maximumRetentionYears = 100
+)
+
+// UnmarshalXML - decodes XML data.
+func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Make subtype to avoid recursive UnmarshalXML().
+	type defaultRetention DefaultRetention
+	retention := defaultRetention{}
+
+	if err := d.DecodeElement(&retention, &start); err != nil {
+		return err
+	}
+
+	switch string(retention.Mode) {
+	case "GOVERNANCE", "COMPLIANCE":
+	default:
+		return fmt.Errorf("unknown retention mode %v", retention.Mode)
+	}
+
+	if retention.Days == nil && retention.Years == nil {
+		return fmt.Errorf("either Days or Years must be specified")
+	}
+
+	if retention.Days != nil && retention.Years != nil {
+		return fmt.Errorf("either Days or Years must be specified, not both")
+	}
+
+	if retention.Days != nil {
+		if *retention.Days == 0 {
+			return fmt.Errorf("Default retention period must be a positive integer value for 'Days'")
+		}
+		if *retention.Days > maximumRetentionDays {
+			return fmt.Errorf("Default retention period too large for 'Days' %d", *retention.Days)
+		}
+	} else if *retention.Years == 0 {
+		return fmt.Errorf("Default retention period must be a positive integer value for 'Years'")
+	} else if *retention.Years > maximumRetentionYears {
+		return fmt.Errorf("Default retention period too large for 'Years' %d", *retention.Years)
+	}
+
+	*dr = DefaultRetention(retention)
+
+	return nil
+}
+
+// Config - object lock configuration specified in
+// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html
+type Config struct {
+	XMLNS             string   `xml:"xmlns,attr,omitempty"`
+	XMLName           xml.Name `xml:"ObjectLockConfiguration"`
+	ObjectLockEnabled string   `xml:"ObjectLockEnabled"`
+	Rule              *struct {
+		DefaultRetention DefaultRetention `xml:"DefaultRetention"`
+	} `xml:"Rule,omitempty"`
+}
+
+// UnmarshalXML - decodes XML data.
+func (config *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
+	// Make subtype to avoid recursive UnmarshalXML().
+	type objectLockConfig Config
+	parsedConfig := objectLockConfig{}
+
+	if err := d.DecodeElement(&parsedConfig, &start); err != nil {
+		return err
+	}
+
+	if parsedConfig.ObjectLockEnabled != "Enabled" {
+		return fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element")
+	}
+
+	*config = Config(parsedConfig)
+	return nil
+}
+
+// ToRetention - convert to Retention type.
+func (config *Config) ToRetention() (r Retention) {
+	if config.Rule != nil {
+		r.Mode = config.Rule.DefaultRetention.Mode
+
+		t, err := UTCNowNTP()
+		if err != nil {
+			logger.LogIf(context.Background(), err)
+			// Do not change any configuration
+			// upon NTP failure.
+			return r
+		}
+
+		if config.Rule.DefaultRetention.Days != nil {
+			r.Validity = t.AddDate(0, 0, int(*config.Rule.DefaultRetention.Days)).Sub(t)
+		} else {
+			r.Validity = t.AddDate(int(*config.Rule.DefaultRetention.Years), 0, 0).Sub(t)
+		}
+	}
+
+	return r
+}
+
+// ParseObjectLockConfig parses ObjectLockConfig from xml
+func ParseObjectLockConfig(reader io.Reader) (*Config, error) {
+	config := Config{}
+	if err := xml.NewDecoder(reader).Decode(&config); err != nil {
+		return nil, err
+	}
+
+	return &config, nil
+}
+
+// NewObjectLockConfig returns a initialized objectlock.Config struct
+func NewObjectLockConfig() *Config {
+	return &Config{
+		ObjectLockEnabled: "Enabled",
+	}
+}
+
+// RetentionDate is a embedded type containing time.Time to unmarshal
+// Date in Retention
+type RetentionDate struct {
+	time.Time
+}
+
+// UnmarshalXML parses date from Retention and validates date format
+func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error {
+	var dateStr string
+	err := d.DecodeElement(&dateStr, &startElement)
+	if err != nil {
+		return err
+	}
+	// While AWS documentation mentions that the date specified
+	// must be present in ISO 8601 format, in reality they allow
+	// users to provide RFC 3339 compliant dates.
+	retDate, err := time.Parse(time.RFC3339, dateStr)
+	if err != nil {
+		return ErrInvalidRetentionDate
+	}
+
+	*rDate = RetentionDate{retDate}
+	return nil
+}
+
+// MarshalXML encodes expiration date if it is non-zero and encodes
+// empty string otherwise
+func (rDate *RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error {
+	if *rDate == (RetentionDate{time.Time{}}) {
+		return nil
+	}
+	return e.EncodeElement(rDate.Format(time.RFC3339), startElement)
+}
+
+// ObjectRetention specified in
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html
+type ObjectRetention struct {
+	XMLNS           string        `xml:"xmlns,attr,omitempty"`
+	XMLName         xml.Name      `xml:"Retention"`
+	Mode            Mode          `xml:"Mode,omitempty"`
+	RetainUntilDate RetentionDate `xml:"RetainUntilDate,omitempty"`
+}
+
+// ParseObjectRetention constructs ObjectRetention struct from xml input
+func ParseObjectRetention(reader io.Reader) (*ObjectRetention, error) {
+	ret := ObjectRetention{}
+	if err := xml.NewDecoder(reader).Decode(&ret); err != nil {
+		return nil, err
+	}
+	if ret.Mode != Compliance && ret.Mode != Governance {
+		return &ret, ErrUnknownWORMModeDirective
+	}
+
+	t, err := UTCNowNTP()
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return &ret, ErrPastObjectLockRetainDate
+	}
+
+	if ret.RetainUntilDate.Before(t) {
+		return &ret, ErrPastObjectLockRetainDate
+	}
+
+	return &ret, nil
+}
+
+// IsObjectLockRetentionRequested returns true if object lock retention headers are set.
+func IsObjectLockRetentionRequested(h http.Header) bool {
+	if _, ok := h[xhttp.AmzObjectLockMode]; ok {
+		return true
+	}
+	if _, ok := h[xhttp.AmzObjectLockRetainUntilDate]; ok {
+		return true
+	}
+	return false
+}
+
+// IsObjectLockLegalHoldRequested returns true if object lock legal hold header is set.
+func IsObjectLockLegalHoldRequested(h http.Header) bool {
+	_, ok := h[xhttp.AmzObjectLockLegalHold]
+	return ok
+}
+
+// IsObjectLockGovernanceBypassSet returns true if object lock governance bypass header is set.
+func IsObjectLockGovernanceBypassSet(h http.Header) bool {
+	return strings.ToLower(h.Get(xhttp.AmzObjectLockBypassGovernance)) == "true"
+}
+
+// IsObjectLockRequested returns true if legal hold or object lock retention headers are requested.
+func IsObjectLockRequested(h http.Header) bool {
+	return IsObjectLockLegalHoldRequested(h) || IsObjectLockRetentionRequested(h)
+}
+
+// ParseObjectLockRetentionHeaders parses http headers to extract retention mode and retention date
+func ParseObjectLockRetentionHeaders(h http.Header) (rmode Mode, r RetentionDate, err error) {
+	retMode := h.Get(xhttp.AmzObjectLockMode)
+	dateStr := h.Get(xhttp.AmzObjectLockRetainUntilDate)
+	if len(retMode) == 0 || len(dateStr) == 0 {
+		return rmode, r, ErrObjectLockInvalidHeaders
+	}
+	rmode = parseMode(retMode)
+	if rmode == Invalid {
+		return rmode, r, ErrUnknownWORMModeDirective
+	}
+
+	var retDate time.Time
+	// While AWS documentation mentions that the date specified
+	// must be present in ISO 8601 format, in reality they allow
+	// users to provide RFC 3339 compliant dates.
+	retDate, err = time.Parse(time.RFC3339, dateStr)
+	if err != nil {
+		return rmode, r, ErrInvalidRetentionDate
+	}
+
+	t, err := UTCNowNTP()
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return rmode, r, ErrPastObjectLockRetainDate
+	}
+
+	if retDate.Before(t) {
+		return rmode, r, ErrPastObjectLockRetainDate
+	}
+
+	return rmode, RetentionDate{retDate}, nil
+
+}
+
+// GetObjectRetentionMeta constructs ObjectRetention from metadata
+func GetObjectRetentionMeta(meta map[string]string) ObjectRetention {
+	var mode Mode
+	var retainTill RetentionDate
+
+	if modeStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockMode)]; ok {
+		mode = parseMode(modeStr)
+	}
+	if tillStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)]; ok {
+		if t, e := time.Parse(time.RFC3339, tillStr); e == nil {
+			retainTill = RetentionDate{t.UTC()}
+		}
+	}
+	return ObjectRetention{Mode: mode, RetainUntilDate: retainTill}
+}
+
+// GetObjectLegalHoldMeta constructs ObjectLegalHold from metadata
+func GetObjectLegalHoldMeta(meta map[string]string) ObjectLegalHold {
+
+	holdStr, ok := meta[strings.ToLower(xhttp.AmzObjectLockLegalHold)]
+	if ok {
+		return ObjectLegalHold{Status: parseLegalHoldStatus(holdStr)}
+	}
+	return ObjectLegalHold{}
+}
+
+// ParseObjectLockLegalHoldHeaders parses request headers to construct ObjectLegalHold
+func ParseObjectLockLegalHoldHeaders(h http.Header) (lhold ObjectLegalHold, err error) {
+	holdStatus, ok := h[xhttp.AmzObjectLockLegalHold]
+	if ok {
+		lh := parseLegalHoldStatus(strings.Join(holdStatus, ""))
+		if lh != ON && lh != OFF {
+			return lhold, ErrUnknownWORMModeDirective
+		}
+		lhold = ObjectLegalHold{Status: lh}
+	}
+	return lhold, nil
+
+}
+
+// ObjectLegalHold specified in
+// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLegalHold.html
+type ObjectLegalHold struct {
+	XMLNS   string          `xml:"xmlns,attr,omitempty"`
+	XMLName xml.Name        `xml:"LegalHold"`
+	Status  LegalHoldStatus `xml:"Status,omitempty"`
+}
+
+// ParseObjectLegalHold decodes the XML into ObjectLegalHold
+func ParseObjectLegalHold(reader io.Reader) (hold *ObjectLegalHold, err error) {
+	if err = xml.NewDecoder(reader).Decode(&hold); err != nil {
+		return
+	}
+
+	if hold.Status != ON && hold.Status != OFF {
+		return nil, ErrMalformedXML
+	}
+	return
+}
+
+// FilterObjectLockMetadata filters object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set.
+func FilterObjectLockMetadata(metadata map[string]string, filterRetention, filterLegalHold bool) map[string]string {
+	// Copy on write
+	dst := metadata
+	var copied bool
+	delKey := func(key string) {
+		if _, ok := metadata[key]; !ok {
+			return
+		}
+		if !copied {
+			dst = make(map[string]string, len(metadata))
+			for k, v := range metadata {
+				dst[k] = v
+			}
+			copied = true
+		}
+		delete(dst, key)
+	}
+	legalHold := GetObjectLegalHoldMeta(metadata)
+	if legalHold.Status == "" || filterLegalHold {
+		delKey(xhttp.AmzObjectLockLegalHold)
+	}
+
+	ret := GetObjectRetentionMeta(metadata)
+
+	if ret.Mode == Invalid || filterRetention {
+		delKey(xhttp.AmzObjectLockMode)
+		delKey(xhttp.AmzObjectLockRetainUntilDate)
+		return dst
+	}
+	return dst
+}

pkg/objectlock/objectlock_test.go

@@ -0,0 +1,567 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package objectlock
+
+import (
+	"encoding/xml"
+	"fmt"
+	"net/http"
+	"reflect"
+	"strings"
+	"testing"
+	"time"
+
+	xhttp "github.com/minio/minio/cmd/http"
+)
+
+func TestParseMode(t *testing.T) {
+	testCases := []struct {
+		value        string
+		expectedMode Mode
+	}{
+		{
+			value:        "governance",
+			expectedMode: Governance,
+		},
+		{
+			value:        "complIAnce",
+			expectedMode: Compliance,
+		},
+		{
+			value:        "gce",
+			expectedMode: Invalid,
+		},
+	}
+
+	for _, tc := range testCases {
+		if parseMode(tc.value) != tc.expectedMode {
+			t.Errorf("Expected Mode %s, got %s", tc.expectedMode, parseMode(tc.value))
+		}
+	}
+}
+func TestParseLegalHoldStatus(t *testing.T) {
+	tests := []struct {
+		value          string
+		expectedStatus LegalHoldStatus
+	}{
+		{
+			value:          "ON",
+			expectedStatus: ON,
+		},
+		{
+			value:          "Off",
+			expectedStatus: OFF,
+		},
+		{
+			value:          "x",
+			expectedStatus: "",
+		},
+	}
+
+	for _, tt := range tests {
+		actualStatus := parseLegalHoldStatus(tt.value)
+		if actualStatus != tt.expectedStatus {
+			t.Errorf("Expected legal hold status %s, got %s", tt.expectedStatus, actualStatus)
+		}
+	}
+}
+
+// TestUnmarshalDefaultRetention checks if default retention
+// marshaling and unmarshaling work as expected
+func TestUnmarshalDefaultRetention(t *testing.T) {
+	days := uint64(4)
+	years := uint64(1)
+	zerodays := uint64(0)
+	invalidDays := uint64(maximumRetentionDays + 1)
+	tests := []struct {
+		value       DefaultRetention
+		expectedErr error
+		expectErr   bool
+	}{
+		{
+			value:       DefaultRetention{Mode: "retain"},
+			expectedErr: fmt.Errorf("unknown retention mode retain"),
+			expectErr:   true,
+		},
+		{
+			value:       DefaultRetention{Mode: "GOVERNANCE"},
+			expectedErr: fmt.Errorf("either Days or Years must be specified"),
+			expectErr:   true,
+		},
+		{
+			value:       DefaultRetention{Mode: "GOVERNANCE", Days: &days},
+			expectedErr: nil,
+			expectErr:   false,
+		},
+		{
+			value:       DefaultRetention{Mode: "GOVERNANCE", Years: &years},
+			expectedErr: nil,
+			expectErr:   false,
+		},
+		{
+			value:       DefaultRetention{Mode: "GOVERNANCE", Days: &days, Years: &years},
+			expectedErr: fmt.Errorf("either Days or Years must be specified, not both"),
+			expectErr:   true,
+		},
+		{
+			value:       DefaultRetention{Mode: "GOVERNANCE", Days: &zerodays},
+			expectedErr: fmt.Errorf("Default retention period must be a positive integer value for 'Days'"),
+			expectErr:   true,
+		},
+		{
+			value:       DefaultRetention{Mode: "GOVERNANCE", Days: &invalidDays},
+			expectedErr: fmt.Errorf("Default retention period too large for 'Days' %d", invalidDays),
+			expectErr:   true,
+		},
+	}
+	for _, tt := range tests {
+		d, err := xml.MarshalIndent(&tt.value, "", "\t")
+		if err != nil {
+			t.Fatal(err)
+		}
+		var dr DefaultRetention
+		err = xml.Unmarshal(d, &dr)
+		if tt.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", tt.expectedErr)
+		} else if tt.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err)
+		}
+	}
+}
+
+func TestParseObjectLockConfig(t *testing.T) {
+	tests := []struct {
+		value       string
+		expectedErr error
+		expectErr   bool
+	}{
+		{
+			value:       "<ObjectLockConfiguration><ObjectLockEnabled>yes</ObjectLockEnabled></ObjectLockConfiguration>",
+			expectedErr: fmt.Errorf("only 'Enabled' value is allowd to ObjectLockEnabled element"),
+			expectErr:   true,
+		},
+		{
+			value:       "<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>0</Days></DefaultRetention></Rule></ObjectLockConfiguration>",
+			expectedErr: fmt.Errorf("Default retention period must be a positive integer value for 'Days'"),
+			expectErr:   true,
+		},
+		{
+			value:       "<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>COMPLIANCE</Mode><Days>30</Days></DefaultRetention></Rule></ObjectLockConfiguration>",
+			expectedErr: nil,
+			expectErr:   false,
+		},
+	}
+	for _, tt := range tests {
+		_, err := ParseObjectLockConfig(strings.NewReader(tt.value))
+		if tt.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", tt.expectedErr)
+		} else if tt.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err)
+		}
+	}
+}
+
+func TestParseObjectRetention(t *testing.T) {
+	tests := []struct {
+		value       string
+		expectedErr error
+		expectErr   bool
+	}{
+		{
+			value:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>string</Mode><RetainUntilDate>2020-01-02T15:04:05Z</RetainUntilDate></Retention>",
+			expectedErr: ErrUnknownWORMModeDirective,
+			expectErr:   true,
+		},
+		{
+			value:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>COMPLIANCE</Mode><RetainUntilDate>2017-01-02T15:04:05Z</RetainUntilDate></Retention>",
+			expectedErr: ErrPastObjectLockRetainDate,
+			expectErr:   true,
+		},
+		{
+			value:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?><Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Mode>GOVERNANCE</Mode><RetainUntilDate>2057-01-02T15:04:05Z</RetainUntilDate></Retention>",
+			expectedErr: nil,
+			expectErr:   false,
+		},
+	}
+	for _, tt := range tests {
+		_, err := ParseObjectRetention(strings.NewReader(tt.value))
+		if tt.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", tt.expectedErr)
+		} else if tt.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err)
+		}
+	}
+}
+
+func TestIsObjectLockRequested(t *testing.T) {
+	tests := []struct {
+		header      http.Header
+		expectedVal bool
+	}{
+		{
+			header: http.Header{
+				"Authorization":        []string{"AWS4-HMAC-SHA256 <cred_string>"},
+				"X-Amz-Content-Sha256": []string{""},
+				"Content-Encoding":     []string{""},
+			},
+			expectedVal: false,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockLegalHold: []string{""},
+			},
+			expectedVal: true,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockRetainUntilDate: []string{""},
+				xhttp.AmzObjectLockMode:            []string{""},
+			},
+			expectedVal: true,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockBypassGovernance: []string{""},
+			},
+			expectedVal: false,
+		},
+	}
+	for _, tt := range tests {
+		actualVal := IsObjectLockRequested(tt.header)
+		if actualVal != tt.expectedVal {
+			t.Fatalf("error: expected %v, actual %v", tt.expectedVal, actualVal)
+		}
+	}
+}
+
+func TestIsObjectLockGovernanceBypassSet(t *testing.T) {
+	tests := []struct {
+		header      http.Header
+		expectedVal bool
+	}{
+		{
+			header: http.Header{
+				"Authorization":        []string{"AWS4-HMAC-SHA256 <cred_string>"},
+				"X-Amz-Content-Sha256": []string{""},
+				"Content-Encoding":     []string{""},
+			},
+			expectedVal: false,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockLegalHold: []string{""},
+			},
+			expectedVal: false,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockRetainUntilDate: []string{""},
+				xhttp.AmzObjectLockMode:            []string{""},
+			},
+			expectedVal: false,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockBypassGovernance: []string{""},
+			},
+			expectedVal: false,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockBypassGovernance: []string{"true"},
+			},
+			expectedVal: true,
+		},
+	}
+	for _, tt := range tests {
+		actualVal := IsObjectLockGovernanceBypassSet(tt.header)
+		if actualVal != tt.expectedVal {
+			t.Fatalf("error: expected %v, actual %v", tt.expectedVal, actualVal)
+		}
+	}
+}
+
+func TestParseObjectLockRetentionHeaders(t *testing.T) {
+	tests := []struct {
+		header      http.Header
+		expectedErr error
+	}{
+		{
+			header: http.Header{
+				"Authorization":        []string{"AWS4-HMAC-SHA256 <cred_string>"},
+				"X-Amz-Content-Sha256": []string{""},
+				"Content-Encoding":     []string{""},
+			},
+			expectedErr: ErrObjectLockInvalidHeaders,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockMode:            []string{"lock"},
+				xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02"},
+			},
+			expectedErr: ErrUnknownWORMModeDirective,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockMode: []string{"governance"},
+			},
+			expectedErr: ErrObjectLockInvalidHeaders,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02"},
+				xhttp.AmzObjectLockMode:            []string{"governance"},
+			},
+			expectedErr: ErrInvalidRetentionDate,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02T15:04:05Z"},
+				xhttp.AmzObjectLockMode:            []string{"governance"},
+			},
+			expectedErr: ErrPastObjectLockRetainDate,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockMode:            []string{"governance"},
+				xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02T15:04:05Z"},
+			},
+			expectedErr: ErrPastObjectLockRetainDate,
+		},
+		{
+			header: http.Header{
+				xhttp.AmzObjectLockMode:            []string{"governance"},
+				xhttp.AmzObjectLockRetainUntilDate: []string{"2087-01-02T15:04:05Z"},
+			},
+			expectedErr: nil,
+		},
+	}
+
+	for i, tt := range tests {
+		_, _, err := ParseObjectLockRetentionHeaders(tt.header)
+		if tt.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("Case %d error: expected = <nil>, got = %v", i, err)
+			}
+		} else if err == nil {
+			t.Fatalf("Case %d error: expected = %v, got = <nil>", i, tt.expectedErr)
+		} else if tt.expectedErr.Error() != err.Error() {
+			t.Fatalf("Case %d error: expected = %v, got = %v", i, tt.expectedErr, err)
+		}
+	}
+}
+
+func TestGetObjectRetentionMeta(t *testing.T) {
+	tests := []struct {
+		metadata map[string]string
+		expected ObjectRetention
+	}{
+		{
+			metadata: map[string]string{
+				"Authorization":        "AWS4-HMAC-SHA256 <cred_string>",
+				"X-Amz-Content-Sha256": "",
+				"Content-Encoding":     "",
+			},
+			expected: ObjectRetention{},
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-mode": "governance",
+			},
+			expected: ObjectRetention{Mode: Governance},
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-retain-until-date": "2020-02-01",
+			},
+			expected: ObjectRetention{RetainUntilDate: RetentionDate{time.Date(2020, 2, 1, 12, 0, 0, 0, time.UTC)}},
+		},
+	}
+
+	for i, tt := range tests {
+		o := GetObjectRetentionMeta(tt.metadata)
+		if o.Mode != tt.expected.Mode {
+			t.Fatalf("Case %d expected %v, got %v", i, tt.expected.Mode, o.Mode)
+		}
+	}
+}
+
+func TestGetObjectLegalHoldMeta(t *testing.T) {
+	tests := []struct {
+		metadata map[string]string
+		expected ObjectLegalHold
+	}{
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-mode": "governance",
+			},
+			expected: ObjectLegalHold{},
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-legal-hold": "on",
+			},
+			expected: ObjectLegalHold{Status: ON},
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-legal-hold": "off",
+			},
+			expected: ObjectLegalHold{Status: OFF},
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-legal-hold": "X",
+			},
+			expected: ObjectLegalHold{Status: ""},
+		},
+	}
+
+	for i, tt := range tests {
+		o := GetObjectLegalHoldMeta(tt.metadata)
+		if o.Status != tt.expected.Status {
+			t.Fatalf("Case %d expected %v, got %v", i, tt.expected.Status, o.Status)
+		}
+	}
+}
+
+func TestParseObjectLegalHold(t *testing.T) {
+	tests := []struct {
+		value       string
+		expectedErr error
+		expectErr   bool
+	}{
+		{
+			value:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>string</Status></LegalHold>",
+			expectedErr: ErrMalformedXML,
+			expectErr:   true,
+		},
+		{
+			value:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>ON</Status></LegalHold>",
+			expectedErr: nil,
+			expectErr:   false,
+		},
+		{
+			value:       "<?xml version=\"1.0\" encoding=\"UTF-8\"?><LegalHold xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\"><Status>On</Status></LegalHold>",
+			expectedErr: ErrMalformedXML,
+			expectErr:   true,
+		},
+	}
+	for i, tt := range tests {
+		_, err := ParseObjectLegalHold(strings.NewReader(tt.value))
+		if tt.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("Case %d error: expected = <nil>, got = %v", i, err)
+			}
+		} else if err == nil {
+			t.Fatalf("Case %d error: expected = %v, got = <nil>", i, tt.expectedErr)
+		} else if tt.expectedErr.Error() != err.Error() {
+			t.Fatalf("Case %d error: expected = %v, got = %v", i, tt.expectedErr, err)
+		}
+	}
+}
+func TestFilterObjectLockMetadata(t *testing.T) {
+	tests := []struct {
+		metadata        map[string]string
+		filterRetention bool
+		filterLegalHold bool
+		expected        map[string]string
+	}{
+		{
+			metadata: map[string]string{
+				"Authorization":        "AWS4-HMAC-SHA256 <cred_string>",
+				"X-Amz-Content-Sha256": "",
+				"Content-Encoding":     "",
+			},
+			expected: map[string]string{
+				"Authorization":        "AWS4-HMAC-SHA256 <cred_string>",
+				"X-Amz-Content-Sha256": "",
+				"Content-Encoding":     "",
+			},
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-mode": "governance",
+			},
+			expected: map[string]string{
+				"x-amz-object-lock-mode": "governance",
+			},
+			filterRetention: false,
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-mode":              "governance",
+				"x-amz-object-lock-retain-until-date": "2020-02-01",
+			},
+			expected:        map[string]string{},
+			filterRetention: true,
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-legal-hold": "off",
+			},
+			expected:        map[string]string{},
+			filterLegalHold: true,
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-legal-hold": "on",
+			},
+			expected:        map[string]string{"x-amz-object-lock-legal-hold": "on"},
+			filterLegalHold: false,
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-legal-hold":        "on",
+				"x-amz-object-lock-mode":              "governance",
+				"x-amz-object-lock-retain-until-date": "2020-02-01",
+			},
+			expected:        map[string]string{},
+			filterRetention: true,
+			filterLegalHold: true,
+		},
+		{
+			metadata: map[string]string{
+				"x-amz-object-lock-legal-hold":        "on",
+				"x-amz-object-lock-mode":              "governance",
+				"x-amz-object-lock-retain-until-date": "2020-02-01",
+			},
+			expected: map[string]string{"x-amz-object-lock-legal-hold": "on",
+				"x-amz-object-lock-mode":              "governance",
+				"x-amz-object-lock-retain-until-date": "2020-02-01"},
+		},
+	}
+
+	for i, tt := range tests {
+		o := FilterObjectLockMetadata(tt.metadata, tt.filterRetention, tt.filterLegalHold)
+		if !reflect.DeepEqual(o, tt.metadata) {
+			t.Fatalf("Case %d expected %v, got %v", i, tt.metadata, o)
+		}
+	}
+}

pkg/policy/action.go

@@ -98,10 +98,21 @@ const (
 
 	// GetObjectRetentionAction - GetObjectRetention, GetObject, HeadObject Rest API action.
 	GetObjectRetentionAction = "s3:GetObjectRetention"
+	// GetObjectLegalHoldAction - GetObjectLegalHold, GetObject Rest API action.
+	GetObjectLegalHoldAction = "s3:GetObjectLegalHold"
+	// PutObjectLegalHoldAction - PutObjectLegalHold, PutObject Rest API action.
+	PutObjectLegalHoldAction = "s3:PutObjectLegalHold"
 	// GetBucketObjectLockConfigurationAction - GetObjectLockConfiguration Rest API action
 	GetBucketObjectLockConfigurationAction = "s3:GetBucketObjectLockConfiguration"
 	// PutBucketObjectLockConfigurationAction - PutObjectLockConfiguration Rest API action
 	PutBucketObjectLockConfigurationAction = "s3:PutBucketObjectLockConfiguration"
+
+	// GetObjectTaggingAction - Get Object Tags API action
+	GetObjectTaggingAction = "s3:GetObjectTagging"
+	// PutObjectTaggingAction - Put Object Tags API action
+	PutObjectTaggingAction = "s3:PutObjectTagging"
+	// DeleteObjectTaggingAction - Delete Object Tags API action
+	DeleteObjectTaggingAction = "s3:DeleteObjectTagging"
 )
 
 // isObjectAction - returns whether action is object type or not.
@@ -113,8 +124,12 @@ func (action Action) isObjectAction() bool {
 		return true
 	case PutObjectRetentionAction, GetObjectRetentionAction:
 		return true
+	case PutObjectLegalHoldAction, GetObjectLegalHoldAction:
+		return true
 	case BypassGovernanceModeAction, BypassGovernanceRetentionAction:
 		return true
+	case GetObjectTaggingAction, PutObjectTaggingAction, DeleteObjectTaggingAction:
+		return true
 	}
 
 	return false
@@ -143,8 +158,12 @@ func (action Action) IsValid() bool {
 		return true
 	case PutObjectRetentionAction, GetObjectRetentionAction:
 		return true
+	case PutObjectLegalHoldAction, GetObjectLegalHoldAction:
+		return true
 	case PutBucketObjectLockConfigurationAction, GetBucketObjectLockConfigurationAction:
 		return true
+	case GetObjectTaggingAction, PutObjectTaggingAction, DeleteObjectTaggingAction:
+		return true
 	}
 
 	return false
@@ -231,6 +250,11 @@ var actionConditionKeyMap = map[Action]condition.KeySet{
 	GetObjectRetentionAction:               condition.NewKeySet(condition.CommonKeys...),
 	BypassGovernanceModeAction:             condition.NewKeySet(condition.CommonKeys...),
 	BypassGovernanceRetentionAction:        condition.NewKeySet(condition.CommonKeys...),
+	PutObjectLegalHoldAction:               condition.NewKeySet(condition.CommonKeys...),
+	GetObjectLegalHoldAction:               condition.NewKeySet(condition.CommonKeys...),
 	GetBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...),
 	PutBucketObjectLockConfigurationAction: condition.NewKeySet(condition.CommonKeys...),
+	PutObjectTaggingAction:                 condition.NewKeySet(condition.CommonKeys...),
+	GetObjectTaggingAction:                 condition.NewKeySet(condition.CommonKeys...),
+	DeleteObjectTaggingAction:              condition.NewKeySet(condition.CommonKeys...),
 }

pkg/tagging/error.go

@@ -0,0 +1,44 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tagging
+
+import (
+	"fmt"
+)
+
+// Error is the generic type for any error happening during tag
+// parsing.
+type Error struct {
+	err error
+}
+
+// Errorf - formats according to a format specifier and returns
+// the string as a value that satisfies error of type tagging.Error
+func Errorf(format string, a ...interface{}) error {
+	return Error{err: fmt.Errorf(format, a...)}
+}
+
+// Unwrap the internal error.
+func (e Error) Unwrap() error { return e.err }
+
+// Error 'error' compatible method.
+func (e Error) Error() string {
+	if e.err == nil {
+		return "tagging: cause <nil>"
+	}
+	return e.err.Error()
+}

pkg/tagging/tag.go

@@ -0,0 +1,62 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tagging
+
+import (
+	"encoding/xml"
+	"unicode/utf8"
+)
+
+// Tag - single tag
+type Tag struct {
+	XMLName xml.Name `xml:"Tag"`
+	Key     string   `xml:"Key"`
+	Value   string   `xml:"Value"`
+}
+
+// Validate - validates the tag element
+func (t Tag) Validate() error {
+	if err := t.validateKey(); err != nil {
+		return err
+	}
+	if err := t.validateValue(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// validateKey - checks if key is valid or not.
+func (t Tag) validateKey() error {
+	// cannot be longer than maxTagKeyLength characters
+	if utf8.RuneCountInString(t.Key) > maxTagKeyLength {
+		return ErrInvalidTagKey
+	}
+	// cannot be empty
+	if len(t.Key) == 0 {
+		return ErrInvalidTagKey
+	}
+	return nil
+}
+
+// validateValue - checks if value is valid or not.
+func (t Tag) validateValue() error {
+	// cannot be longer than maxTagValueLength characters
+	if utf8.RuneCountInString(t.Value) > maxTagValueLength {
+		return ErrInvalidTagValue
+	}
+	return nil
+}

pkg/tagging/tagging.go

@@ -0,0 +1,113 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tagging
+
+import (
+	"bytes"
+	"encoding/xml"
+	"io"
+	"net/url"
+)
+
+// S3 API limits for tags
+// Ref: https://docs.aws.amazon.com/AmazonS3/latest/dev/object-tagging.html
+const (
+	maxTags           = 10
+	maxTagKeyLength   = 128
+	maxTagValueLength = 256
+)
+
+// errors returned by tagging package
+var (
+	ErrTooManyTags     = Errorf("Cannot have more than 10 object tags")
+	ErrInvalidTagKey   = Errorf("The TagKey you have provided is invalid")
+	ErrInvalidTagValue = Errorf("The TagValue you have provided is invalid")
+	ErrInvalidTag      = Errorf("Cannot provide multiple Tags with the same key")
+)
+
+// Tagging - object tagging interface
+type Tagging struct {
+	XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Tagging"`
+	TagSet  TagSet   `xml:"TagSet"`
+}
+
+// Validate - validates the tagging configuration
+func (t Tagging) Validate() error {
+	// Tagging can't have more than 10 tags
+	if len(t.TagSet.Tags) > maxTags {
+		return ErrTooManyTags
+	}
+	// Validate all the rules in the tagging config
+	for _, ts := range t.TagSet.Tags {
+		if t.TagSet.ContainsDuplicate(ts.Key) {
+			return ErrInvalidTag
+		}
+		if err := ts.Validate(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// String - returns a string in format "tag1=value1&tag2=value2" with all the
+// tags in this Tagging Struct
+func (t Tagging) String() string {
+	var buf bytes.Buffer
+	for _, tag := range t.TagSet.Tags {
+		if buf.Len() > 0 {
+			buf.WriteString("&")
+		}
+		buf.WriteString(tag.Key + "=")
+		buf.WriteString(tag.Value)
+	}
+	return buf.String()
+}
+
+// FromString - returns a Tagging struct when given a string in format
+// "tag1=value1&tag2=value2"
+func FromString(tagStr string) (Tagging, error) {
+	tags, err := url.ParseQuery(tagStr)
+	if err != nil {
+		return Tagging{}, err
+	}
+	var idx = 0
+	parsedTags := make([]Tag, len(tags))
+	for k := range tags {
+		parsedTags[idx].Key = k
+		parsedTags[idx].Value = tags.Get(k)
+		idx++
+	}
+	return Tagging{
+		TagSet: TagSet{
+			Tags: parsedTags,
+		},
+	}, nil
+}
+
+// ParseTagging - parses incoming xml data in given reader
+// into Tagging interface. After parsing, also validates the
+// parsed fields based on S3 API constraints.
+func ParseTagging(reader io.Reader) (*Tagging, error) {
+	var t Tagging
+	if err := xml.NewDecoder(reader).Decode(&t); err != nil {
+		return nil, err
+	}
+	if err := t.Validate(); err != nil {
+		return nil, err
+	}
+	return &t, nil
+}

pkg/tagging/tagset.go

@@ -0,0 +1,41 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package tagging
+
+import (
+	"encoding/xml"
+)
+
+// TagSet - Set of tags under Tagging
+type TagSet struct {
+	XMLName xml.Name `xml:"TagSet"`
+	Tags    []Tag    `xml:"Tag"`
+}
+
+// ContainsDuplicate - returns true if duplicate keys are present in TagSet
+func (t TagSet) ContainsDuplicate(key string) bool {
+	var found bool
+	for _, tag := range t.Tags {
+		if tag.Key == key {
+			if found {
+				return true
+			}
+			found = true
+		}
+	}
+	return false
+}