Allow querying a single target in config get API (#15587)

This is needed to ensure that we do not leave prefixes where
version is suspended, instead we never leave versions on
these paths.

.github/workflows/go-fips.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.11b7, 1.18.3b7]
+        go-version: [1.18.5b7]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2

.nancy-ignore

@@ -2,3 +2,5 @@ CVE-2020-26160
 CVE-2020-15136
 CVE-2020-15115
 CVE-2020-15114
+CVE-2022-2835
+CVE-2022-2837

buildscripts/unaligned-healing.sh

@@ -120,7 +120,7 @@ function start_minio_16drive() {
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
 
 	"${WORK_DIR}/mc" mb play/inspects
 	"${WORK_DIR}/mc" mirror inspects play/inspects
@@ -140,7 +140,7 @@ function start_minio_16drive() {
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
 
 	"${WORK_DIR}/mc" mb play/inspects
 	"${WORK_DIR}/mc" mirror inspects play/inspects

cmd/admin-bucket-handlers.go

@@ -249,7 +249,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	}
 	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil {
 		switch err.(type) {
-		case BucketRemoteConnectionErr:
+		case RemoteTargetConnectionErr:
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
@@ -845,7 +845,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		}
 		switch fileName {
 		case bucketNotificationConfig:
-			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalNotificationSys.targetList)
+			config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region, globalEventNotifier.targetList)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err))
 				continue
@@ -862,7 +862,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 			rulesMap := config.ToRulesMap()
-			globalNotificationSys.AddRulesMap(bucket, rulesMap)
+			globalEventNotifier.AddRulesMap(bucket, rulesMap)
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.

cmd/admin-handlers-config-kv.go

@@ -177,6 +177,11 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 }
 
 // GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
+//
+// `key` can be one of three forms:
+// 1. `subsys:target` -> request for config of a single subsystem and target pair.
+// 2. `subsys:` -> request for config of a single subsystem and the default target.
+// 3. `subsys` -> request for config of all targets for the given subsystem.
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
 
@@ -189,8 +194,22 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 
 	cfg := globalServerConfig.Clone()
 	vars := mux.Vars(r)
-	subSys := vars["key"]
-	subSysConfigs, err := cfg.GetSubsysInfo(subSys)
+	key := vars["key"]
+
+	var subSys, target string
+	{
+		ws := strings.SplitN(key, madmin.SubSystemSeparator, 2)
+		subSys = ws[0]
+		if len(ws) == 2 {
+			if ws[1] == "" {
+				target = madmin.Default
+			} else {
+				target = ws[1]
+			}
+		}
+	}
+
+	subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -433,27 +452,27 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	}
 	for _, hkv := range hkvs {
 		// We ignore the error below, as we cannot get one.
-		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key)
+		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
 
 		for _, item := range cfgSubsysItems {
-			off := item.Params.Get(config.Enable) == config.EnableOff
+			off := item.Config.Get(config.Enable) == config.EnableOff
 			switch hkv.Key {
 			case config.EtcdSubSys:
-				off = !etcd.Enabled(item.Params)
+				off = !etcd.Enabled(item.Config)
 			case config.CacheSubSys:
-				off = !cache.Enabled(item.Params)
+				off = !cache.Enabled(item.Config)
 			case config.StorageClassSubSys:
-				off = !storageclass.Enabled(item.Params)
+				off = !storageclass.Enabled(item.Config)
 			case config.PolicyPluginSubSys:
-				off = !polplugin.Enabled(item.Params)
+				off = !polplugin.Enabled(item.Config)
 			case config.IdentityOpenIDSubSys:
-				off = !openid.Enabled(item.Params)
+				off = !openid.Enabled(item.Config)
 			case config.IdentityLDAPSubSys:
-				off = !xldap.Enabled(item.Params)
+				off = !xldap.Enabled(item.Config)
 			case config.IdentityTLSSubSys:
 				off = !globalSTSTLSConfig.Enabled
 			case config.IdentityPluginSubSys:
-				off = !idplugin.Enabled(item.Params)
+				off = !idplugin.Enabled(item.Config)
 			}
 			item.AddString(&s, off)
 		}

cmd/admin-handlers-users.go

@@ -1529,7 +1529,29 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		}
 	}
 
-	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, isGroup)
+	// Validate that user or group exists.
+	if !isGroup {
+		if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
+			_, ok := globalIAMSys.GetUser(ctx, entityName)
+			if !ok {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+				return
+			}
+		}
+	} else {
+		_, err := globalIAMSys.GetGroupDescription(entityName)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
+	userType := regUser
+	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
+		userType = stsUser
+	}
+
+	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, userType, isGroup)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1539,6 +1561,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: entityName,
+			UserType:    int(userType),
 			IsGroup:     isGroup,
 			Policy:      policyName,
 		},
@@ -2114,7 +2137,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, userPolicyMappingsFile, u), r.URL)
 					return
 				}
-				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL)
 					return
 				}
@@ -2143,7 +2166,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for g, pm := range grpPolicyMap {
-				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, g), r.URL)
 					return
 				}
@@ -2182,7 +2205,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL)
 					return
 				}
-				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, false); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL)
 					return
 				}
@@ -2211,7 +2234,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for g, pm := range grpPolicyMap {
-				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, true); err != nil {
+				if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil {
 					writeErrorResponseJSON(ctx, w, importError(ctx, err, stsGroupPolicyMappingsFile, g), r.URL)
 					return
 				}

cmd/admin-handlers.go

@@ -398,8 +398,21 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 	} else {
 		types = madmin.MetricsAll
 	}
+
+	disks := strings.Split(r.Form.Get("disks"), ",")
+	byDisk := strings.EqualFold(r.Form.Get("by-disk"), "true")
+	var diskMap map[string]struct{}
+	if len(disks) > 0 && disks[0] != "" {
+		diskMap = make(map[string]struct{}, len(disks))
+		for _, k := range disks {
+			if k != "" {
+				diskMap[k] = struct{}{}
+			}
+		}
+	}
+
 	hosts := strings.Split(r.Form.Get("hosts"), ",")
-	byhost := strings.EqualFold(r.Form.Get("by-host"), "true")
+	byHost := strings.EqualFold(r.Form.Get("by-host"), "true")
 	var hostMap map[string]struct{}
 	if len(hosts) > 0 && hosts[0] != "" {
 		hostMap = make(map[string]struct{}, len(hosts))
@@ -409,6 +422,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 			}
 		}
 	}
+
 	done := ctx.Done()
 	ticker := time.NewTicker(interval)
 	defer ticker.Stop()
@@ -416,17 +430,20 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 
 	for n > 0 {
 		var m madmin.RealtimeMetrics
-		mLocal := collectLocalMetrics(types, hostMap)
+		mLocal := collectLocalMetrics(types, hostMap, diskMap)
 		m.Merge(&mLocal)
 
 		// Allow half the interval for collecting remote...
 		cctx, cancel := context.WithTimeout(ctx, interval/2)
-		mRemote := collectRemoteMetrics(cctx, types, hostMap)
+		mRemote := collectRemoteMetrics(cctx, types, hostMap, diskMap)
 		cancel()
 		m.Merge(&mRemote)
-		if !byhost {
+		if !byHost {
 			m.ByHost = nil
 		}
+		if !byDisk {
+			m.ByDisk = nil
+		}
 
 		m.Final = n <= 1
 		// Marshal API response
@@ -1862,7 +1879,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 		Mode:         string(mode),
 		Domain:       domain,
 		Region:       globalSite.Region,
-		SQSARN:       globalNotificationSys.GetARNList(false),
+		SQSARN:       globalEventNotifier.GetARNList(false),
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
 		Objects:      objects,

cmd/api-errors.go

@@ -196,8 +196,9 @@ const (
 	ErrInvalidTagDirective
 	// Add new error codes here.
 
-	// SSE-S3 related API errors
+	// SSE-S3/SSE-KMS related API errors
 	ErrInvalidEncryptionMethod
+	ErrInvalidEncryptionKeyID
 
 	// Server-Side-Encryption (with Customer provided key) related API errors.
 	ErrInsecureSSECustomerRequest
@@ -887,7 +888,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrReplicationRemoteConnectionError: {
 		Code:           "XMinioAdminReplicationRemoteConnectionError",
-		Description:    "Remote service connection error - please check remote service credentials and target bucket",
+		Description:    "Remote service connection error",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationBandwidthLimitError: {
@@ -1072,6 +1073,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The encryption method specified is not supported",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidEncryptionKeyID: {
+		Code:           "InvalidRequest",
+		Description:    "The specified KMS KeyID contains unsupported characters",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInsecureSSECustomerRequest: {
 		Code:           "InvalidRequest",
 		Description:    "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.",
@@ -1921,6 +1927,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrInvalidEncryptionParameters
 	case crypto.ErrInvalidEncryptionMethod:
 		apiErr = ErrInvalidEncryptionMethod
+	case crypto.ErrInvalidEncryptionKeyID:
+		apiErr = ErrInvalidEncryptionKeyID
 	case crypto.ErrInvalidCustomerAlgorithm:
 		apiErr = ErrInvalidSSECustomerAlgorithm
 	case crypto.ErrMissingCustomerKey:
@@ -2066,7 +2074,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrRemoteDestinationNotFoundError
 	case BucketRemoteTargetNotFound:
 		apiErr = ErrRemoteTargetNotFoundError
-	case BucketRemoteConnectionErr:
+	case RemoteTargetConnectionErr:
 		apiErr = ErrReplicationRemoteConnectionError
 	case BucketRemoteAlreadyExists:
 		apiErr = ErrBucketRemoteAlreadyExists

cmd/api-headers.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/json"
-	"encoding/xml"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -30,6 +29,8 @@ import (
 
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	xxml "github.com/minio/xxml"
 )
 
 // Returns a hexadecimal representation of time at the
@@ -64,9 +65,13 @@ func setCommonHeaders(w http.ResponseWriter) {
 // Encodes the response headers into XML format.
 func encodeResponse(response interface{}) []byte {
 	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xml.Header)
-	e := xml.NewEncoder(&bytesBuffer)
-	e.Encode(response)
+	bytesBuffer.WriteString(xxml.Header)
+	buf, err := xxml.Marshal(response)
+	if err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	bytesBuffer.Write(buf)
 	return bytesBuffer.Bytes()
 }
 

cmd/api-response.go

@@ -89,7 +89,8 @@ type ListVersionsResponse struct {
 	IsTruncated bool
 
 	CommonPrefixes []CommonPrefix
-	Versions       []ObjectVersion
+	DeleteMarkers  []DeleteMarkerVersion `xml:"DeleteMarker,omitempty"`
+	Versions       []ObjectVersion       `xml:"Version,omitempty"`
 
 	// Encoding type used to encode object keys in the response.
 	EncodingType string `xml:"EncodingType,omitempty"`
@@ -247,50 +248,80 @@ type ObjectVersion struct {
 	Object
 	IsLatest  bool
 	VersionID string `xml:"VersionId"`
-
-	isDeleteMarker bool
 }
 
-// MarshalXML - marshal ObjectVersion
-func (o ObjectVersion) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	if o.isDeleteMarker {
-		start.Name.Local = "DeleteMarker"
-	} else {
-		start.Name.Local = "Version"
+// DeleteMarkerVersion container for delete marker metadata
+type DeleteMarkerVersion struct {
+	Key          string
+	LastModified string // time string of format "2006-01-02T15:04:05.000Z"
+
+	// Owner of the object.
+	Owner Owner
+
+	IsLatest  bool
+	VersionID string `xml:"VersionId"`
+}
+
+// Metadata metadata items implemented to ensure XML marshaling works.
+type Metadata struct {
+	Items []struct {
+		Key   string
+		Value string
 	}
-
-	type objectVersionWrapper ObjectVersion
-	return e.EncodeElement(objectVersionWrapper(o), start)
 }
 
-// StringMap is a map[string]string
-type StringMap map[string]string
+// Set add items, duplicate items get replaced.
+func (s *Metadata) Set(k, v string) {
+	for i, item := range s.Items {
+		if item.Key == k {
+			s.Items[i] = struct {
+				Key   string
+				Value string
+			}{
+				Key:   k,
+				Value: v,
+			}
+			return
+		}
+	}
+	s.Items = append(s.Items, struct {
+		Key   string
+		Value string
+	}{
+		Key:   k,
+		Value: v,
+	})
+}
+
+type xmlKeyEntry struct {
+	XMLName xml.Name
+	Value   string `xml:",chardata"`
+}
 
 // MarshalXML - StringMap marshals into XML.
-func (s StringMap) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
-	tokens := []xml.Token{start}
-
-	for key, value := range s {
-		t := xml.StartElement{}
-		t.Name = xml.Name{
-			Space: "",
-			Local: key,
-		}
-		tokens = append(tokens, t, xml.CharData(value), xml.EndElement{Name: t.Name})
+func (s *Metadata) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+	if s == nil {
+		return nil
 	}
 
-	tokens = append(tokens, xml.EndElement{
-		Name: start.Name,
-	})
+	if len(s.Items) == 0 {
+		return nil
+	}
 
-	for _, t := range tokens {
-		if err := e.EncodeToken(t); err != nil {
+	if err := e.EncodeToken(start); err != nil {
+		return err
+	}
+
+	for _, item := range s.Items {
+		if err := e.Encode(xmlKeyEntry{
+			XMLName: xml.Name{Local: item.Key},
+			Value:   item.Value,
+		}); err != nil {
 			return err
 		}
 	}
 
-	// flush to ensure tokens are written
-	return e.Flush()
+	return e.EncodeToken(start.End())
 }
 
 // Object container for object metadata
@@ -307,7 +338,7 @@ type Object struct {
 	StorageClass string
 
 	// UserMetadata user-defined metadata
-	UserMetadata StringMap `xml:"UserMetadata,omitempty"`
+	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
 }
 
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -438,6 +469,8 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
+	deleteMarkers := make([]DeleteMarkerVersion, 0, len(resp.Objects))
+
 	owner := Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
@@ -445,10 +478,25 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 	data := ListVersionsResponse{}
 
 	for _, object := range resp.Objects {
-		content := ObjectVersion{}
 		if object.Name == "" {
 			continue
 		}
+
+		if object.DeleteMarker {
+			deleteMarker := DeleteMarkerVersion{
+				Key:          s3EncodeName(object.Name, encodingType),
+				LastModified: object.ModTime.UTC().Format(iso8601TimeFormat),
+				Owner:        owner,
+				VersionID:    object.VersionID,
+			}
+			if deleteMarker.VersionID == "" {
+				deleteMarker.VersionID = nullVersionID
+			}
+			deleteMarker.IsLatest = object.IsLatest
+			deleteMarkers = append(deleteMarkers, deleteMarker)
+			continue
+		}
+		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
 		if object.ETag != "" {
@@ -466,12 +514,12 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 			content.VersionID = nullVersionID
 		}
 		content.IsLatest = object.IsLatest
-		content.isDeleteMarker = object.DeleteMarker
 		versions = append(versions, content)
 	}
 
 	data.Name = bucket
 	data.Versions = versions
+	data.DeleteMarkers = deleteMarkers
 	data.EncodingType = encodingType
 	data.Prefix = s3EncodeName(prefix, encodingType)
 	data.KeyMarker = s3EncodeName(marker, encodingType)
@@ -569,14 +617,14 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 		}
 		content.Owner = owner
 		if metadata {
-			content.UserMetadata = make(StringMap)
+			content.UserMetadata = &Metadata{}
 			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
 			case crypto.S3:
-				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 			case crypto.S3KMS:
-				content.UserMetadata[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
 			case crypto.SSEC:
-				content.UserMetadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
+				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
 			for k, v := range CleanMinioInternalMetadataKeys(object.UserDefined) {
 				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
@@ -588,7 +636,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 					continue
 				}
-				content.UserMetadata[k] = v
+				content.UserMetadata.Set(k, v)
 			}
 		}
 		contents = append(contents, content)

cmd/bucket-metadata-sys.go

@@ -452,7 +452,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 			sys.metadataMap[buckets[index].Name] = meta
 			sys.Unlock()
 
-			globalNotificationSys.set(buckets[index], meta) // set notification targets
+			globalEventNotifier.set(buckets[index], meta) // set notification targets
 
 			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
 

cmd/bucket-notification-handlers.go

@@ -72,7 +72,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 	config.SetRegion(globalSite.Region)
-	if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
+	if err = config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
 		if ok {
 			for i, queue := range config.QueueList {
@@ -144,7 +144,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalEventNotifier.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {
@@ -166,7 +166,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 	}
 
 	rulesMap := config.ToRulesMap()
-	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
+	globalEventNotifier.AddRulesMap(bucketName, rulesMap)
 
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/bucket-replication-handlers.go

@@ -310,7 +310,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 	rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID})
 	if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil {
 		switch err.(type) {
-		case BucketRemoteConnectionErr:
+		case RemoteTargetConnectionErr:
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL)
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)

cmd/bucket-replication-utils.go

@@ -737,3 +737,32 @@ func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) {
 	opts.Prefix = q.Get("prefix")
 	return
 }
+
+const (
+	replicationMRFDir = bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "mrf"
+	mrfMetaFormat     = 1
+	mrfMetaVersionV1  = 1
+	mrfMetaVersion    = mrfMetaVersionV1
+)
+
+// MRFReplicateEntry mrf entry to save to disk
+type MRFReplicateEntry struct {
+	Bucket    string `json:"bucket" msg:"b"`
+	Object    string `json:"object" msg:"o"`
+	versionID string `json:"-"`
+}
+
+// MRFReplicateEntries has the map of MRF entries to save to disk
+type MRFReplicateEntries struct {
+	Entries map[string]MRFReplicateEntry `json:"entries" msg:"e"`
+	Version int                          `json:"version" msg:"v"`
+}
+
+// ToMRFEntry returns the relevant info needed by MRF
+func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
+	return MRFReplicateEntry{
+		Bucket:    ri.Bucket,
+		Object:    ri.Name,
+		versionID: ri.VersionID,
+	}
+}

cmd/bucket-replication-utils_gen.go

@@ -260,6 +260,413 @@ func (z *BucketReplicationResyncStatus) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "e":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Entries")
+				return
+			}
+			if z.Entries == nil {
+				z.Entries = make(map[string]MRFReplicateEntry, zb0002)
+			} else if len(z.Entries) > 0 {
+				for key := range z.Entries {
+					delete(z.Entries, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 MRFReplicateEntry
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Entries")
+					return
+				}
+				var zb0003 uint32
+				zb0003, err = dc.ReadMapHeader()
+				if err != nil {
+					err = msgp.WrapError(err, "Entries", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, err = dc.ReadMapKeyPtr()
+					if err != nil {
+						err = msgp.WrapError(err, "Entries", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "b":
+						za0002.Bucket, err = dc.ReadString()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+							return
+						}
+					case "o":
+						za0002.Object, err = dc.ReadString()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Object")
+							return
+						}
+					default:
+						err = dc.Skip()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001)
+							return
+						}
+					}
+				}
+				z.Entries[za0001] = za0002
+			}
+		case "v":
+			z.Version, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "e"
+	err = en.Append(0x82, 0xa1, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.Entries)))
+	if err != nil {
+		err = msgp.WrapError(err, "Entries")
+		return
+	}
+	for za0001, za0002 := range z.Entries {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries")
+			return
+		}
+		// map header, size 2
+		// write "b"
+		err = en.Append(0x82, 0xa1, 0x62)
+		if err != nil {
+			return
+		}
+		err = en.WriteString(za0002.Bucket)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+			return
+		}
+		// write "o"
+		err = en.Append(0xa1, 0x6f)
+		if err != nil {
+			return
+		}
+		err = en.WriteString(za0002.Object)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries", za0001, "Object")
+			return
+		}
+	}
+	// write "v"
+	err = en.Append(0xa1, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Version)
+	if err != nil {
+		err = msgp.WrapError(err, "Version")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "e"
+	o = append(o, 0x82, 0xa1, 0x65)
+	o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
+	for za0001, za0002 := range z.Entries {
+		o = msgp.AppendString(o, za0001)
+		// map header, size 2
+		// string "b"
+		o = append(o, 0x82, 0xa1, 0x62)
+		o = msgp.AppendString(o, za0002.Bucket)
+		// string "o"
+		o = append(o, 0xa1, 0x6f)
+		o = msgp.AppendString(o, za0002.Object)
+	}
+	// string "v"
+	o = append(o, 0xa1, 0x76)
+	o = msgp.AppendInt(o, z.Version)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "e":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Entries")
+				return
+			}
+			if z.Entries == nil {
+				z.Entries = make(map[string]MRFReplicateEntry, zb0002)
+			} else if len(z.Entries) > 0 {
+				for key := range z.Entries {
+					delete(z.Entries, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 MRFReplicateEntry
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Entries")
+					return
+				}
+				var zb0003 uint32
+				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Entries", za0001)
+					return
+				}
+				for zb0003 > 0 {
+					zb0003--
+					field, bts, err = msgp.ReadMapKeyZC(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Entries", za0001)
+						return
+					}
+					switch msgp.UnsafeString(field) {
+					case "b":
+						za0002.Bucket, bts, err = msgp.ReadStringBytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Bucket")
+							return
+						}
+					case "o":
+						za0002.Object, bts, err = msgp.ReadStringBytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "Object")
+							return
+						}
+					default:
+						bts, err = msgp.Skip(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001)
+							return
+						}
+					}
+				}
+				z.Entries[za0001] = za0002
+			}
+		case "v":
+			z.Version, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Version")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *MRFReplicateEntries) Msgsize() (s int) {
+	s = 1 + 2 + msgp.MapHeaderSize
+	if z.Entries != nil {
+		for za0001, za0002 := range z.Entries {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object)
+		}
+	}
+	s += 2 + msgp.IntSize
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "b":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "o":
+			z.Object, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "b"
+	err = en.Append(0x82, 0xa1, 0x62)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "o"
+	err = en.Append(0xa1, 0x6f)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Object)
+	if err != nil {
+		err = msgp.WrapError(err, "Object")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "b"
+	o = append(o, 0x82, 0xa1, 0x62)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "o"
+	o = append(o, 0xa1, 0x6f)
+	o = msgp.AppendString(o, z.Object)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "b":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "o":
+			z.Object, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z MRFReplicateEntry) Msgsize() (s int) {
+	s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object)
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/bucket-replication-utils_gen_test.go

@@ -122,6 +122,232 @@ func BenchmarkDecodeBucketReplicationResyncStatus(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalMRFReplicateEntries(t *testing.T) {
+	v := MRFReplicateEntries{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeMRFReplicateEntries(t *testing.T) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeMRFReplicateEntries Msgsize() is inaccurate")
+	}
+
+	vn := MRFReplicateEntries{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeMRFReplicateEntries(b *testing.B) {
+	v := MRFReplicateEntries{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalMRFReplicateEntry(t *testing.T) {
+	v := MRFReplicateEntry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeMRFReplicateEntry(t *testing.T) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeMRFReplicateEntry Msgsize() is inaccurate")
+	}
+
+	vn := MRFReplicateEntry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeMRFReplicateEntry(b *testing.B) {
+	v := MRFReplicateEntry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalReplicateDecision(t *testing.T) {
 	v := ReplicateDecision{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-replication.go

@@ -197,7 +197,7 @@ func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplica
 
 	// Disable server-side replication on object prefixes which are excluded
 	// from versioning via the MinIO bucket versioning extension.
-	if globalBucketVersioningSys.PrefixSuspended(bucket, object) {
+	if !globalBucketVersioningSys.PrefixEnabled(bucket, object) {
 		return
 	}
 
@@ -403,6 +403,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
+		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
 		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, rcfg.RoleArn))
 		sendEvent(eventArgs{
 			BucketName: bucket,
@@ -477,6 +478,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	eventName := event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
+		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
 	}
 	drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
 	if replicationStatus != prevStatus {
@@ -488,7 +490,11 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		MTime:             dobj.DeleteMarkerMTime.Time,
 		DeleteReplication: drs,
 		Versioned:         globalBucketVersioningSys.PrefixEnabled(bucket, dobj.ObjectName),
-		VersionSuspended:  globalBucketVersioningSys.PrefixSuspended(bucket, dobj.ObjectName),
+		// Objects matching prefixes should not leave delete markers,
+		// dramatically reduces namespace pollution while keeping the
+		// benefits of replication, make sure to apply version suspension
+		// only at bucket level instead.
+		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
 		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
@@ -534,7 +540,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 	if dobj.VersionID != "" && rinfo.VersionPurgeStatus == Complete {
 		return
 	}
-	if tgt.IsOffline() {
+	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN))
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
@@ -910,6 +916,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			Object:     objInfo,
 			Host:       "Internal: [Replication]",
 		})
+		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, cfg.RoleArn))
 		return
 	}
@@ -992,12 +999,12 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 
 	// re-queue failures once more - keep a retry count to avoid flooding the queue if
 	// the target site is down. Leave it to scanner to catch up instead.
-	if rinfos.ReplicationStatus() != replication.Completed && ri.RetryCount < 1 {
+	if rinfos.ReplicationStatus() != replication.Completed {
 		ri.OpType = replication.HealReplicationType
 		ri.EventType = ReplicateMRF
 		ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal()
 		ri.RetryCount++
-		globalReplicationPool.queueReplicaFailedTask(ri)
+		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 	}
 }
 
@@ -1039,7 +1046,7 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 		rinfo.ReplicationResynced = true
 		return
 	}
-	if tgt.IsOffline() {
+	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
@@ -1059,13 +1066,15 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 		VersionSuspended: versionSuspended,
 	})
 	if err != nil {
-		sendEvent(eventArgs{
-			EventName:  event.ObjectReplicationNotTracked,
-			BucketName: bucket,
-			Object:     objInfo,
-			Host:       "Internal: [Replication]",
-		})
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replicate for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+		if !isErrObjectNotFound(err) {
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				Host:       "Internal: [Replication]",
+			})
+			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+		}
 		return
 	}
 	defer func() {
@@ -1302,6 +1311,19 @@ type DeletedObjectReplicationInfo struct {
 	TargetArn string
 }
 
+// ToMRFEntry returns the relevant info needed by MRF
+func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry {
+	versionID := di.DeleteMarkerVersionID
+	if versionID == "" {
+		versionID = di.VersionID
+	}
+	return MRFReplicateEntry{
+		Bucket:    di.Bucket,
+		Object:    di.ObjectName,
+		versionID: versionID,
+	}
+}
+
 // Replication specific APIName
 const (
 	ReplicateObjectAPI = "ReplicateObject"
@@ -1346,13 +1368,16 @@ type ReplicationPool struct {
 	mrfReplicaCh            chan ReplicateObjectInfo
 	existingReplicaCh       chan ReplicateObjectInfo
 	existingReplicaDeleteCh chan DeletedObjectReplicationInfo
-	workerSize              int
-	mrfWorkerSize           int
-	resyncState             replicationResyncState
-	workerWg                sync.WaitGroup
-	mrfWorkerWg             sync.WaitGroup
-	once                    sync.Once
-	mu                      sync.Mutex
+	mrfSaveCh               chan MRFReplicateEntry
+
+	workerSize    int
+	mrfWorkerSize int
+	resyncState   replicationResyncState
+	workerWg      sync.WaitGroup
+	mrfWorkerWg   sync.WaitGroup
+	once          sync.Once
+	mu            sync.Mutex
+	mrfMutex      sync.Mutex
 }
 
 // NewReplicationPool creates a pool of replication workers of specified size
@@ -1366,6 +1391,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		existingReplicaCh:       make(chan ReplicateObjectInfo, 100000),
 		existingReplicaDeleteCh: make(chan DeletedObjectReplicationInfo, 100000),
 		resyncState:             replicationResyncState{statusMap: make(map[string]BucketReplicationResyncStatus)},
+		mrfSaveCh:               make(chan MRFReplicateEntry, 100000),
 		ctx:                     ctx,
 		objLayer:                o,
 	}
@@ -1374,6 +1400,8 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	pool.ResizeFailedWorkers(opts.FailedWorkers)
 	go pool.AddExistingObjectReplicateWorker()
 	go pool.updateResyncStatus(ctx, o)
+	go pool.processMRF()
+	go pool.persistMRF()
 	return pool
 }
 
@@ -1479,33 +1507,17 @@ func (p *ReplicationPool) suggestedWorkers(failQueue bool) int {
 	return int(float64(p.workerSize) * ReplicationWorkerMultiplier)
 }
 
-func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
-	if p == nil {
-		return
-	}
-	select {
-	case <-GlobalContext.Done():
-		p.once.Do(func() {
-			close(p.replicaCh)
-			close(p.mrfReplicaCh)
-			close(p.existingReplicaCh)
-		})
-	case p.mrfReplicaCh <- ri:
-	default:
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), string(replicationSubsystem))
-	}
-}
-
 func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
 	}
-	var ch chan ReplicateObjectInfo
+	var ch, healCh chan ReplicateObjectInfo
 	switch ri.OpType {
 	case replication.ExistingObjectReplicationType:
 		ch = p.existingReplicaCh
 	case replication.HealReplicationType:
-		fallthrough
+		ch = p.mrfReplicaCh
+		healCh = p.replicaCh
 	default:
 		ch = p.replicaCh
 	}
@@ -1516,6 +1528,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 			close(p.mrfReplicaCh)
 			close(p.existingReplicaCh)
 		})
+	case healCh <- ri:
 	case ch <- ri:
 	default:
 		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
@@ -1659,7 +1672,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 	}
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, t.Arn)
-		if tgt == nil || tgt.IsOffline() {
+		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			continue
 		}
 		// if proxying explicitly disabled on remote target
@@ -2053,9 +2066,9 @@ func resyncBucket(ctx context.Context, bucket, arn string, heal bool, objectAPI
 		return
 	}
 
-	// Walk through all object versions - note ascending order of walk needed to ensure delete marker replicated to
-	// target after object version is first created.
-	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{WalkAscending: true}); err != nil {
+	// Walk through all object versions - Walk() is always in ascending order needed to ensure
+	// delete marker replicated to target after object version is first created.
+	if err := objectAPI.Walk(ctx, bucket, "", objInfoCh, ObjectOptions{}); err != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
@@ -2386,3 +2399,279 @@ func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string,
 	}()
 	return diffCh, nil
 }
+
+// QueueReplicationHeal is a wrapper for queueReplicationHeal
+func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo) {
+	// un-versioned case
+	if oi.VersionID == "" {
+		return
+	}
+	rcfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	queueReplicationHeal(ctx, bucket, oi, replicationConfig{
+		Config:  rcfg,
+		remotes: tgts,
+	})
+}
+
+// queueReplicationHeal enqueues objects that failed replication OR eligible for resyncing through
+// an ongoing resync operation or via existing objects replication configuration setting.
+func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcfg replicationConfig) (roi ReplicateObjectInfo) {
+	// un-versioned case
+	if oi.VersionID == "" {
+		return roi
+	}
+
+	if rcfg.Config == nil || rcfg.remotes == nil {
+		return roi
+	}
+	roi = getHealReplicateObjectInfo(oi, rcfg)
+	if !roi.Dsc.ReplicateAny() {
+		return
+	}
+	// early return if replication already done, otherwise we need to determine if this
+	// version is an existing object that needs healing.
+	if oi.ReplicationStatus == replication.Completed && oi.VersionPurgeStatus.Empty() && !roi.ExistingObjResync.mustResync() {
+		return
+	}
+
+	if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
+		versionID := ""
+		dmVersionID := ""
+		if roi.VersionPurgeStatus.Empty() {
+			dmVersionID = roi.VersionID
+		} else {
+			versionID = roi.VersionID
+		}
+
+		dv := DeletedObjectReplicationInfo{
+			DeletedObject: DeletedObject{
+				ObjectName:            roi.Name,
+				DeleteMarkerVersionID: dmVersionID,
+				VersionID:             versionID,
+				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+				DeleteMarker:          roi.DeleteMarker,
+			},
+			Bucket:    roi.Bucket,
+			OpType:    replication.HealReplicationType,
+			EventType: ReplicateHealDelete,
+		}
+		// heal delete marker replication failure or versioned delete replication failure
+		if roi.ReplicationStatus == replication.Pending ||
+			roi.ReplicationStatus == replication.Failed ||
+			roi.VersionPurgeStatus == Failed || roi.VersionPurgeStatus == Pending {
+			globalReplicationPool.queueReplicaDeleteTask(dv)
+			return
+		}
+		// if replication status is Complete on DeleteMarker and existing object resync required
+		if roi.ExistingObjResync.mustResync() && (roi.ReplicationStatus == replication.Completed || roi.ReplicationStatus.Empty()) {
+			queueReplicateDeletesWrapper(dv, roi.ExistingObjResync)
+			return
+		}
+		return
+	}
+	if roi.ExistingObjResync.mustResync() {
+		roi.OpType = replication.ExistingObjectReplicationType
+	}
+	switch roi.ReplicationStatus {
+	case replication.Pending, replication.Failed:
+		roi.EventType = ReplicateHeal
+		globalReplicationPool.queueReplicaTask(roi)
+		return
+	}
+	if roi.ExistingObjResync.mustResync() {
+		roi.EventType = ReplicateExisting
+		globalReplicationPool.queueReplicaTask(roi)
+	}
+	return
+}
+
+const mrfTimeInterval = 5 * time.Minute
+
+func (p *ReplicationPool) persistMRF() {
+	var mu sync.Mutex
+	entries := make(map[string]MRFReplicateEntry)
+	mTimer := time.NewTimer(mrfTimeInterval)
+	defer mTimer.Stop()
+	saveMRFToDisk := func(drain bool) {
+		mu.Lock()
+		defer mu.Unlock()
+		if len(entries) == 0 {
+			return
+		}
+		cctx := p.ctx
+		if drain {
+			cctx = context.Background()
+			// drain all mrf entries and save to disk
+			for e := range p.mrfSaveCh {
+				entries[e.versionID] = e
+			}
+		}
+		if err := p.saveMRFEntries(cctx, entries); err != nil {
+			logger.LogOnceIf(p.ctx, fmt.Errorf("Unable to persist replication failures to disk:%w", err), string(replicationSubsystem))
+		}
+		entries = make(map[string]MRFReplicateEntry)
+		return
+	}
+	for {
+		select {
+		case <-mTimer.C:
+			saveMRFToDisk(false)
+			mTimer.Reset(mrfTimeInterval)
+		case <-p.ctx.Done():
+			close(p.mrfSaveCh)
+			saveMRFToDisk(true)
+			return
+		case e, ok := <-p.mrfSaveCh:
+			if !ok {
+				return
+			}
+			var cnt int
+			mu.Lock()
+			entries[e.versionID] = e
+			cnt = len(entries)
+			mu.Unlock()
+			if cnt >= cap(p.mrfSaveCh) || len(p.mrfSaveCh) >= int(0.8*float32(cap(p.mrfSaveCh))) {
+				saveMRFToDisk(true)
+			}
+		}
+	}
+}
+
+func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
+	if p == nil {
+		return
+	}
+	select {
+	case <-GlobalContext.Done():
+		return
+	case p.mrfSaveCh <- entry:
+	}
+}
+
+// save mrf entries to mrf_<uuid>.bin
+func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string]MRFReplicateEntry) error {
+	if len(entries) == 0 {
+		return nil
+	}
+	v := MRFReplicateEntries{
+		Entries: entries,
+		Version: mrfMetaVersionV1,
+	}
+	data := make([]byte, 4, v.Msgsize()+4)
+
+	// Initialize the resync meta header.
+	binary.LittleEndian.PutUint16(data[0:2], resyncMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], resyncMetaVersion)
+
+	buf, err := v.MarshalMsg(data)
+	if err != nil {
+		return err
+	}
+
+	configFile := path.Join(replicationMRFDir, mustGetUUID()+".bin")
+	err = saveConfig(ctx, p.objLayer, configFile, buf)
+	return err
+}
+
+// load mrf entries from disk
+func (p *ReplicationPool) loadMRF(fileName string) (re MRFReplicateEntries, e error) {
+	data, err := readConfig(p.ctx, p.objLayer, fileName)
+	if err != nil && err != errConfigNotFound {
+		return re, err
+	}
+	if len(data) == 0 {
+		// Seems to be empty.
+		return re, nil
+	}
+	if len(data) <= 4 {
+		return re, fmt.Errorf("replication mrf: no data")
+	}
+	// Read resync meta header
+	switch binary.LittleEndian.Uint16(data[0:2]) {
+	case mrfMetaFormat:
+	default:
+		return re, fmt.Errorf("replication mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+	}
+	switch binary.LittleEndian.Uint16(data[2:4]) {
+	case mrfMetaVersion:
+	default:
+		return re, fmt.Errorf("replication mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+	}
+	// OK, parse data.
+	if _, err = re.UnmarshalMsg(data[4:]); err != nil {
+		return re, err
+	}
+
+	switch re.Version {
+	case mrfMetaVersionV1:
+	default:
+		return re, fmt.Errorf("unexpected mrf meta version: %d", re.Version)
+	}
+	return re, nil
+}
+
+func (p *ReplicationPool) processMRF() {
+	if p == nil || p.objLayer == nil {
+		return
+	}
+	pTimer := time.NewTimer(mrfTimeInterval)
+	defer pTimer.Stop()
+	for {
+		select {
+		case <-pTimer.C:
+			// skip healing if all targets are offline
+			var offlineCnt int
+			tgts := globalBucketTargetSys.ListTargets(p.ctx, "", "")
+			for _, tgt := range tgts {
+				if globalBucketTargetSys.isOffline(tgt.URL()) {
+					offlineCnt++
+				}
+			}
+			if len(tgts) == offlineCnt {
+				pTimer.Reset(mrfTimeInterval)
+				continue
+			}
+			objCh := make(chan ObjectInfo)
+			cctx, cancelFn := context.WithCancel(p.ctx)
+			if err := p.objLayer.Walk(cctx, minioMetaBucket, replicationMRFDir, objCh, ObjectOptions{}); err != nil {
+				pTimer.Reset(mrfTimeInterval)
+				cancelFn()
+				logger.LogIf(p.ctx, err)
+				continue
+			}
+			for item := range objCh {
+				if err := p.queueMRFHeal(item.Name); err == nil {
+					p.objLayer.DeleteObject(p.ctx, minioMetaBucket, item.Name, ObjectOptions{})
+				}
+			}
+			pTimer.Reset(mrfTimeInterval)
+			cancelFn()
+		case <-p.ctx.Done():
+			return
+		}
+	}
+}
+
+// process sends error logs to the heal channel for an attempt to heal replication.
+func (p *ReplicationPool) queueMRFHeal(file string) error {
+	if p == nil || p.objLayer == nil {
+		return errServerNotInitialized
+	}
+
+	mrfRec, err := p.loadMRF(file)
+	if err != nil {
+		return err
+	}
+	for vID, e := range mrfRec.Entries {
+		oi, err := p.objLayer.GetObjectInfo(p.ctx, e.Bucket, e.Object, ObjectOptions{
+			VersionID: vID,
+		})
+		if err != nil {
+			continue
+		}
+		QueueReplicationHeal(p.ctx, e.Bucket, oi)
+	}
+	return nil
+}

cmd/bucket-targets.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"net/url"
 	"sync"
 	"time"
 
@@ -35,7 +37,9 @@ import (
 )
 
 const (
-	defaultHealthCheckDuration = 30 * time.Second
+	defaultHealthCheckDuration = 5 * time.Second
+	// default interval for reload of all remote target endpoints
+	defaultHealthCheckReloadDuration = 30 * time.Minute
 )
 
 // BucketTargetSys represents bucket targets subsystem
@@ -43,6 +47,113 @@ type BucketTargetSys struct {
 	sync.RWMutex
 	arnRemotesMap map[string]*TargetClient
 	targetsMap    map[string][]madmin.BucketTarget
+	hMutex        sync.RWMutex
+	hc            map[string]epHealth
+	hcClient      *madmin.AnonymousClient
+}
+
+// epHealth struct represents health of a replication target endpoint.
+type epHealth struct {
+	Endpoint string
+	Scheme   string
+	Online   bool
+}
+
+// isOffline returns current liveness result of remote target. Add endpoint to
+// healthcheck map if missing and default to online status
+func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
+	sys.hMutex.RLock()
+	defer sys.hMutex.RUnlock()
+	if h, ok := sys.hc[ep.Host]; ok {
+		return !h.Online
+	}
+	go sys.initHC(ep)
+	return false
+}
+
+func (sys *BucketTargetSys) initHC(ep *url.URL) {
+	sys.hMutex.Lock()
+	sys.hc[ep.Host] = epHealth{
+		Endpoint: ep.Host,
+		Scheme:   ep.Scheme,
+		Online:   true,
+	}
+	sys.hMutex.Unlock()
+}
+
+// newHCClient initializes an anonymous client for performing health check on the remote endpoints
+func newHCClient() *madmin.AnonymousClient {
+	clnt, e := madmin.NewAnonymousClientNoEndpoint()
+	if e != nil {
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to initialize health check client"), string(replicationSubsystem))
+		return nil
+	}
+	clnt.SetCustomTransport(globalRemoteTargetTransport)
+	return clnt
+}
+
+// heartBeat performs liveness check on remote endpoints.
+func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
+	hcTimer := time.NewTimer(defaultHealthCheckDuration)
+	defer hcTimer.Stop()
+	for {
+		select {
+		case <-hcTimer.C:
+			sys.hMutex.RLock()
+			var eps []madmin.ServerProperties
+			for _, ep := range sys.hc {
+				eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme})
+			}
+			sys.hMutex.RUnlock()
+
+			if len(eps) > 0 {
+				cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
+				defer cancel()
+				m := map[string]epHealth{}
+				for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
+					var online bool
+					if result.Error == nil {
+						online = result.Online
+					}
+					m[result.Endpoint.Host] = epHealth{
+						Endpoint: result.Endpoint.Host,
+						Scheme:   result.Endpoint.Scheme,
+						Online:   online,
+					}
+				}
+				sys.hMutex.Lock()
+				sys.hc = m
+				sys.hMutex.Unlock()
+			}
+			hcTimer.Reset(defaultHealthCheckDuration)
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// periodically rebuild the healthcheck map from list of targets to clear
+// out stale endpoints
+func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) {
+	m := make(map[string]epHealth)
+	tgts := sys.ListTargets(ctx, "", "")
+	for _, t := range tgts {
+		if _, ok := m[t.Endpoint]; !ok {
+			scheme := "http"
+			if t.Secure {
+				scheme = "https"
+			}
+			m[t.Endpoint] = epHealth{
+				Online:   true,
+				Endpoint: t.Endpoint,
+				Scheme:   scheme,
+			}
+		}
+	}
+	sys.hMutex.Lock()
+	// swap out the map
+	sys.hc = m
+	sys.hMutex.Unlock()
 }
 
 // ListTargets lists bucket targets across tenant or for individual bucket, and returns
@@ -91,9 +202,6 @@ func (sys *BucketTargetSys) Delete(bucket string) {
 		return
 	}
 	for _, t := range tgts {
-		if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
-			tgt.healthCancelFn()
-		}
 		delete(sys.arnRemotesMap, t.Arn)
 	}
 	delete(sys.targetsMap, bucket)
@@ -116,7 +224,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
 			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 		}
-		return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 	}
 	if tgt.Type == madmin.ReplicationService {
 		if !globalBucketVersioningSys.Enabled(bucket) {
@@ -124,7 +232,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
 		if err != nil {
-			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
 		}
 		if vcfg.Status != string(versioning.Enabled) {
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
@@ -152,10 +260,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	if !found && !update {
 		newtgts = append(newtgts, *tgt)
 	}
-	// cancel health check for previous target client to avoid leak.
-	if prevClnt, ok := sys.arnRemotesMap[tgt.Arn]; ok && prevClnt.healthCancelFn != nil {
-		prevClnt.healthCancelFn()
-	}
+
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
 	sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
@@ -227,9 +332,6 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 		return BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	sys.targetsMap[bucket] = targets
-	if tgt, ok := sys.arnRemotesMap[arnStr]; ok && tgt.healthCancelFn != nil {
-		tgt.healthCancelFn()
-	}
 	delete(sys.arnRemotesMap, arnStr)
 	sys.updateBandwidthLimit(bucket, 0)
 	return nil
@@ -258,11 +360,29 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
 }
 
 // NewBucketTargetSys - creates new replication system.
-func NewBucketTargetSys() *BucketTargetSys {
-	return &BucketTargetSys{
+func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
+	sys := &BucketTargetSys{
 		arnRemotesMap: make(map[string]*TargetClient),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
+		hc:            make(map[string]epHealth),
+		hcClient:      newHCClient(),
 	}
+	// reload healthcheck endpoints map periodically to remove stale endpoints from the map.
+	go func() {
+		rTimer := time.NewTimer(defaultHealthCheckReloadDuration)
+		defer rTimer.Stop()
+		for {
+			select {
+			case <-rTimer.C:
+				sys.reloadHealthCheckers(ctx)
+				rTimer.Reset(defaultHealthCheckReloadDuration)
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	go sys.heartBeat(ctx)
+	return sys
 }
 
 // UpdateAllTargets updates target to reflect metadata updates
@@ -276,9 +396,6 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 	// Remove existingtarget and arn association
 	if tgts, ok := sys.targetsMap[bucket]; ok {
 		for _, t := range tgts {
-			if tgt, ok := sys.arnRemotesMap[t.Arn]; ok && tgt.healthCancelFn != nil {
-				tgt.healthCancelFn()
-			}
 			delete(sys.arnRemotesMap, t.Arn)
 		}
 		delete(sys.targetsMap, bucket)
@@ -345,10 +462,6 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
 	}
-	cancelFn, err := api.HealthCheck(hcDuration)
-	if err != nil {
-		return nil, err
-	}
 	tc := &TargetClient{
 		Client:              api,
 		healthCheckDuration: hcDuration,
@@ -356,9 +469,10 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 		Bucket:              tcfg.TargetBucket,
 		StorageClass:        tcfg.StorageClass,
 		disableProxy:        tcfg.DisableProxy,
-		healthCancelFn:      cancelFn,
 		ARN:                 tcfg.Arn,
 		ResetID:             tcfg.ResetID,
+		Endpoint:            tcfg.Endpoint,
+		Secure:              tcfg.Secure,
 	}
 	return tc, nil
 }
@@ -432,7 +546,8 @@ type TargetClient struct {
 	replicateSync       bool
 	StorageClass        string // storage class on remote
 	disableProxy        bool
-	healthCancelFn      context.CancelFunc // cancellation function for client healthcheck
-	ARN                 string             // ARN to uniquely identify remote target
+	ARN                 string // ARN to uniquely identify remote target
 	ResetID             string
+	Endpoint            string
+	Secure              bool
 }

cmd/config-current.go

@@ -396,7 +396,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	}
 
 	if config.NotifySubSystems.Contains(subSys) {
-		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalNotificationSys.ConfiguredTargetIDs(), subSys); err != nil {
+		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), globalEventNotifier.ConfiguredTargetIDs(), subSys); err != nil {
 			return err
 		}
 	}
@@ -471,14 +471,12 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if etcdCfg.Enabled {
-		if globalEtcdClient == nil {
-			globalEtcdClient, err = etcd.New(etcdCfg)
-			if err != nil {
-				if globalIsGateway {
-					logger.FatalIf(err, "Unable to initialize etcd config")
-				} else {
-					logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
-				}
+		globalEtcdClient, err = etcd.New(etcdCfg)
+		if err != nil {
+			if globalIsGateway {
+				logger.FatalIf(err, "Unable to initialize etcd config")
+			} else {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
 			}
 		}
 
@@ -486,7 +484,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			if globalDNSConfig != nil {
 				// if global DNS is already configured, indicate with a warning, incase
 				// users are confused.
-				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, not using etcd for DNS store", globalDNSConfig))
+				logger.LogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig))
 			} else {
 				globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config,
 					dns.DomainNames(globalDomainNames),
@@ -518,16 +516,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
-	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err))
-	}
-
-	// Initialize remote instance transport once.
-	getRemoteInstanceTransportOnce.Do(func() {
-		getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
-	})
-
 	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
 	if err != nil {
 		if globalIsGateway {
@@ -560,56 +548,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
-	globalOpenIDConfig, err = openid.LookupConfig(s,
-		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
-	}
-
-	globalLDAPConfig, err = xldap.Lookup(s[config.IdentityLDAPSubSys][config.Default],
-		globalRootCAs)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
-	}
-
-	authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
-	}
-	globalAuthNPlugin = idplugin.New(authNPluginCfg)
-
-	authZPluginCfg, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody)
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err))
-	}
-	if authZPluginCfg.URL == nil {
-		opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
-			NewGatewayHTTPTransport(), xhttp.DrainBody)
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
-		} else {
-			authZPluginCfg.URL = opaCfg.URL
-			authZPluginCfg.AuthToken = opaCfg.AuthToken
-			authZPluginCfg.Transport = opaCfg.Transport
-			authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
-		}
-	}
-
-	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
-
 	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalProxyTransport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
 	}
 
-	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
+	transport := NewGatewayHTTPTransport()
+
+	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, transport, false)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), NewGatewayHTTPTransport(), true)
+	globalEnvTargetList, err = notify.GetNotificationTargets(GlobalContext, newServerConfig(), transport, true)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
@@ -636,6 +587,11 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			setDriveCounts = objAPI.SetDriveCounts()
 		}
 		globalAPIConfig.init(apiConfig, setDriveCounts)
+
+		// Initialize remote instance transport once.
+		getRemoteInstanceTransportOnce.Do(func() {
+			getRemoteInstanceTransport = newGatewayHTTPTransport(apiConfig.RemoteTransportDeadline)
+		})
 	case config.CompressionSubSys:
 		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 		if err != nil {
@@ -678,8 +634,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.HTTP[n] = l
 			}
 		}
-		err = logger.UpdateSystemTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateSystemTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %w", err))
 		}
 	case config.AuditWebhookSubSys:
@@ -697,8 +652,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 		}
 
-		err = logger.UpdateAuditWebhookTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateAuditWebhookTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %w", err))
 		}
 	case config.AuditKafkaSubSys:
@@ -712,8 +666,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		err = logger.UpdateAuditKafkaTargets(loggerCfg)
-		if err != nil {
+		if err = logger.UpdateAuditKafkaTargets(loggerCfg); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %w", err))
 		}
 	case config.StorageClassSubSys:
@@ -852,7 +805,7 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	globalServerConfigMu.Unlock()
 
 	// Save config into file.
-	return saveServerConfig(GlobalContext, objAPI, globalServerConfig)
+	return saveServerConfig(GlobalContext, objAPI, srvCfg)
 }
 
 func getValidConfig(objAPI ObjectLayer) (config.Config, error) {

cmd/data-scanner.go

@@ -64,7 +64,11 @@ const (
 var (
 	globalHealConfig heal.Config
 
-	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	dataScannerLeaderLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
+		timeout:       30 * time.Second,
+		minimum:       10 * time.Second,
+		retryInterval: time.Second,
+	})
 	// Sleeper values are updated when config is loaded.
 	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
 	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
@@ -1225,29 +1229,16 @@ func (i *scannerItem) objectPath() string {
 
 // healReplication will heal a scanned item that has failed replication.
 func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
-	roi := getHealReplicateObjectInfo(oi, i.replication)
-	if !roi.Dsc.ReplicateAny() {
+	if oi.VersionID == "" {
 		return
 	}
-
+	if i.replication.Config == nil {
+		return
+	}
+	roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication)
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
-		// heal delete marker replication failure or versioned delete replication failure
-		if oi.ReplicationStatus == replication.Pending ||
-			oi.ReplicationStatus == replication.Failed ||
-			oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
-			i.healReplicationDeletes(ctx, o, roi)
-			return
-		}
-		// if replication status is Complete on DeleteMarker and existing object resync required
-		if roi.ExistingObjResync.mustResync() && (oi.ReplicationStatus == replication.Completed || oi.ReplicationStatus.Empty()) {
-			i.healReplicationDeletes(ctx, o, roi)
-			return
-		}
 		return
 	}
-	if roi.ExistingObjResync.mustResync() {
-		roi.OpType = replication.ExistingObjectReplicationType
-	}
 
 	if sizeS.replTargetStats == nil && len(roi.TargetStatuses) > 0 {
 		sizeS.replTargetStats = make(map[string]replTargetSizeSummary)
@@ -1277,52 +1268,9 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 	}
 
 	switch oi.ReplicationStatus {
-	case replication.Pending, replication.Failed:
-		roi.EventType = ReplicateHeal
-		globalReplicationPool.queueReplicaTask(roi)
-		return
 	case replication.Replica:
 		sizeS.replicaSize += oi.Size
 	}
-	if roi.ExistingObjResync.mustResync() {
-		roi.EventType = ReplicateExisting
-		globalReplicationPool.queueReplicaTask(roi)
-	}
-}
-
-// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
-func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, roi ReplicateObjectInfo) {
-	// handle soft delete and permanent delete failures here.
-	if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
-		versionID := ""
-		dmVersionID := ""
-		if roi.VersionPurgeStatus.Empty() {
-			dmVersionID = roi.VersionID
-		} else {
-			versionID = roi.VersionID
-		}
-
-		doi := DeletedObjectReplicationInfo{
-			DeletedObject: DeletedObject{
-				ObjectName:            roi.Name,
-				DeleteMarkerVersionID: dmVersionID,
-				VersionID:             versionID,
-				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
-				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-				DeleteMarker:          roi.DeleteMarker,
-			},
-			Bucket:    roi.Bucket,
-			OpType:    replication.HealReplicationType,
-			EventType: ReplicateHealDelete,
-		}
-		if roi.ExistingObjResync.mustResync() {
-			doi.OpType = replication.ExistingObjectReplicationType
-			doi.EventType = ReplicateExistingDelete
-			queueReplicateDeletesWrapper(doi, roi.ExistingObjResync)
-			return
-		}
-		globalReplicationPool.queueReplicaDeleteTask(doi)
-	}
 }
 
 type dynamicSleeper struct {

cmd/decommetric_string.go

@@ -0,0 +1,25 @@
+// Code generated by "stringer -type=decomMetric -trimprefix=decomMetric erasure-server-pool-decom.go"; DO NOT EDIT.
+
+package cmd
+
+import "strconv"
+
+func _() {
+	// An "invalid array index" compiler error signifies that the constant values have changed.
+	// Re-run the stringer command to generate them again.
+	var x [1]struct{}
+	_ = x[decomMetricDecommissionBucket-0]
+	_ = x[decomMetricDecommissionObject-1]
+	_ = x[decomMetricDecommissionRemoveObject-2]
+}
+
+const _decomMetric_name = "DecommissionBucketDecommissionObjectDecommissionRemoveObject"
+
+var _decomMetric_index = [...]uint8{0, 18, 36, 60}
+
+func (i decomMetric) String() string {
+	if i >= decomMetric(len(_decomMetric_index)-1) {
+		return "decomMetric(" + strconv.FormatInt(int64(i), 10) + ")"
+	}
+	return _decomMetric_name[_decomMetric_index[i]:_decomMetric_index[i+1]]
+}

cmd/dynamic-timeouts.go

@@ -34,11 +34,24 @@ const (
 
 // timeouts that are dynamically adapted based on actual usage results
 type dynamicTimeout struct {
-	timeout int64
-	minimum int64
-	entries int64
-	log     [dynamicTimeoutLogSize]time.Duration
-	mutex   sync.Mutex
+	timeout       int64
+	minimum       int64
+	entries       int64
+	log           [dynamicTimeoutLogSize]time.Duration
+	mutex         sync.Mutex
+	retryInterval time.Duration
+}
+
+type dynamicTimeoutOpts struct {
+	timeout       time.Duration
+	minimum       time.Duration
+	retryInterval time.Duration
+}
+
+func newDynamicTimeoutWithOpts(opts dynamicTimeoutOpts) *dynamicTimeout {
+	dt := newDynamicTimeout(opts.timeout, opts.minimum)
+	dt.retryInterval = opts.retryInterval
+	return dt
 }
 
 // newDynamicTimeout returns a new dynamic timeout initialized with timeout value
@@ -57,6 +70,10 @@ func (dt *dynamicTimeout) Timeout() time.Duration {
 	return time.Duration(atomic.LoadInt64(&dt.timeout))
 }
 
+func (dt *dynamicTimeout) RetryInterval() time.Duration {
+	return dt.retryInterval
+}
+
 // LogSuccess logs the duration of a successful action that
 // did not hit the timeout
 func (dt *dynamicTimeout) LogSuccess(duration time.Duration) {

cmd/encryption-v1.go

@@ -82,7 +82,7 @@ func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDe
 // metadata, if any. It returns an empty ID if no key ID is
 // present.
 func kmsKeyIDFromMetadata(metadata map[string]string) string {
-	const ARNPrefix = "arn:aws:kms:"
+	const ARNPrefix = crypto.ARNPrefix
 	if len(metadata) == 0 {
 		return ""
 	}

cmd/erasure-object.go

@@ -448,15 +448,26 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		}
 		defer NSUpdated(bucket, object)
 
-		if opts.VersionID != "" {
-			er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
-				VersionID: opts.VersionID,
-			}, false)
-		} else {
-			er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
-				VersionID: m.VersionID,
-			}, false)
+		fi := FileInfo{
+			VersionID: m.VersionID,
 		}
+		if opts.VersionID != "" {
+			fi.VersionID = opts.VersionID
+		}
+
+		disks := er.getDisks()
+		g := errgroup.WithNErrs(len(disks))
+		for index := range disks {
+			index := index
+			g.Go(func() error {
+				if disks[index] == nil {
+					return errDiskNotFound
+				}
+				return disks[index].DeleteVersion(ctx, bucket, object, fi, false)
+			}, index)
+		}
+
+		g.Wait()
 	}
 	return m, err
 }
@@ -1172,8 +1183,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
-func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo, forceDelMarker bool) error {
+func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker bool) error {
 	disks := er.getDisks()
+	// Assume (N/2 + 1) quorum for Delete()
+	// this is a theoretical assumption such that
+	// for delete's we do not need to honor storage
+	// class for objects that have reduced quorum
+	// due to storage class - this only needs to be honored
+	// for Read() requests alone that we already do.
+	writeQuorum := len(disks)/2 + 1
+
 	g := errgroup.WithNErrs(len(disks))
 	for index := range disks {
 		index := index
@@ -1423,69 +1442,49 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	// Acquire a write lock before deleting the object.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
-	if err != nil {
-		return ObjectInfo{}, err
-	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
-
-	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
-	goi, writeQuorum, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
-	if gerr != nil && goi.Name == "" {
-		switch gerr.(type) {
-		case InsufficientReadQuorum:
-			return objInfo, InsufficientWriteQuorum{}
-		}
-		// For delete marker replication, versionID being replicated will not exist on disk
-		if opts.DeleteMarker {
-			versionFound = false
-		} else {
-			return objInfo, gerr
-		}
-	}
-
-	if opts.Expiration.Expire {
-		action := evalActionFromLifecycle(ctx, *lc, rcfg, goi, false)
-		var isErr bool
-		switch action {
-		case lifecycle.NoneAction:
-			isErr = true
-		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			isErr = true
-		}
-		if isErr {
-			if goi.VersionID != "" {
-				return goi, VersionNotFound{
-					Bucket:    bucket,
-					Object:    object,
-					VersionID: goi.VersionID,
-				}
-			}
-			return goi, ObjectNotFound{
-				Bucket: bucket,
-				Object: object,
-			}
-		}
-	}
-
 	defer NSUpdated(bucket, object)
 
 	storageDisks := er.getDisks()
 
-	var markDelete bool
-	// Determine whether to mark object deleted for replication
-	if goi.VersionID != "" {
-		markDelete = true
+	if opts.Expiration.Expire {
+		goi, _, err := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
+		if err == nil {
+			action := evalActionFromLifecycle(ctx, *lc, rcfg, goi, false)
+			var isErr bool
+			switch action {
+			case lifecycle.NoneAction:
+				isErr = true
+			case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+				isErr = true
+			}
+			if isErr {
+				if goi.VersionID != "" {
+					return goi, VersionNotFound{
+						Bucket:    bucket,
+						Object:    object,
+						VersionID: goi.VersionID,
+					}
+				}
+				return goi, ObjectNotFound{
+					Bucket: bucket,
+					Object: object,
+				}
+			}
+		}
 	}
 
-	// Default deleteMarker to true if object is under versioning
-	deleteMarker := opts.Versioned
+	versionFound := !(opts.DeleteMarker && opts.VersionID != "")
 
-	if opts.VersionID != "" {
+	// Determine whether to mark object deleted for replication
+	markDelete := !opts.DeleteMarker && opts.VersionID != ""
+
+	// Default deleteMarker to true if object is under versioning
+	// versioning suspended means we add `null` version as
+	// delete marker, if its not decided already.
+	deleteMarker := (opts.Versioned || opts.VersionSuspended) && opts.VersionID == "" || (opts.DeleteMarker && opts.VersionID != "")
+
+	if markDelete {
 		// case where replica version needs to be deleted on target cluster
 		if versionFound && opts.DeleteMarkerReplicationStatus() == replication.Replica {
 			markDelete = false
@@ -1496,22 +1495,6 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		if opts.VersionPurgeStatus() == Complete {
 			markDelete = false
 		}
-
-		// Version is found but we do not wish to create more delete markers
-		// now, since VersionPurgeStatus() is already set, we can let the
-		// lower layers decide this. This fixes a regression that was introduced
-		// in PR #14555 where !VersionPurgeStatus.Empty() is automatically
-		// considered as Delete marker true to avoid listing such objects by
-		// regular ListObjects() calls. However for delete replication this
-		// ends up being a problem because "upon" a successful delete this
-		// ends up creating a new delete marker that is spurious and unnecessary.
-		if versionFound {
-			if !goi.VersionPurgeStatus.Empty() {
-				deleteMarker = false
-			} else if !goi.DeleteMarker { // implies a versioned delete of object
-				deleteMarker = false
-			}
-		}
 	}
 
 	modTime := opts.MTime
@@ -1519,38 +1502,32 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		modTime = UTCNow()
 	}
 	fvID := mustGetUUID()
-	if markDelete {
-		if opts.Versioned || opts.VersionSuspended {
-			if !deleteMarker {
-				// versioning suspended means we add `null` version as
-				// delete marker, if its not decided already.
-				deleteMarker = opts.VersionSuspended && opts.VersionID == ""
-			}
-			fi := FileInfo{
-				Name:             object,
-				Deleted:          deleteMarker,
-				MarkDeleted:      markDelete,
-				ModTime:          modTime,
-				ReplicationState: opts.DeleteReplication,
-				TransitionStatus: opts.Transition.Status,
-				ExpireRestored:   opts.Transition.ExpireRestored,
-			}
-			fi.SetTierFreeVersionID(fvID)
-			if opts.Versioned {
-				fi.VersionID = mustGetUUID()
-				if opts.VersionID != "" {
-					fi.VersionID = opts.VersionID
-				}
-			}
-			// versioning suspended means we add `null` version as
-			// delete marker. Add delete marker, since we don't have
-			// any version specified explicitly. Or if a particular
-			// version id needs to be replicated.
-			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
-				return objInfo, toObjectErr(err, bucket, object)
-			}
-			return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
+
+	if markDelete && (opts.Versioned || opts.VersionSuspended) {
+		fi := FileInfo{
+			Name:             object,
+			Deleted:          true,
+			MarkDeleted:      markDelete,
+			ModTime:          modTime,
+			ReplicationState: opts.DeleteReplication,
+			TransitionStatus: opts.Transition.Status,
+			ExpireRestored:   opts.Transition.ExpireRestored,
 		}
+		fi.SetTierFreeVersionID(fvID)
+		if opts.Versioned {
+			fi.VersionID = mustGetUUID()
+			if opts.VersionID != "" {
+				fi.VersionID = opts.VersionID
+			}
+		}
+		// versioning suspended means we add `null` version as
+		// delete marker. Add delete marker, since we don't have
+		// any version specified explicitly. Or if a particular
+		// version id needs to be replicated.
+		if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
+			return objInfo, toObjectErr(err, bucket, object)
+		}
+		return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 	}
 
 	// Delete the object version on all disks.
@@ -1565,7 +1542,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		ExpireRestored:   opts.Transition.ExpireRestored,
 	}
 	dfi.SetTierFreeVersionID(fvID)
-	if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, dfi, opts.DeleteMarker); err != nil {
+	if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 
@@ -1577,13 +1554,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		break
 	}
 
-	return ObjectInfo{
-		Bucket:                     bucket,
-		Name:                       object,
-		VersionID:                  opts.VersionID,
-		VersionPurgeStatusInternal: opts.DeleteReplication.VersionPurgeStatusInternal,
-		ReplicationStatusInternal:  opts.DeleteReplication.ReplicationStatusInternal,
-	}, nil
+	return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
 // Send the successful but partial upload/delete, however ignore
@@ -1853,14 +1824,8 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	eventName := event.ObjectTransitionComplete
 
 	storageDisks := er.getDisks()
-	// we now know the number of blocks this object needs for data and parity.
-	// writeQuorum is dataBlocks + 1
-	writeQuorum := fi.Erasure.DataBlocks
-	if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
-		writeQuorum++
-	}
 
-	if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, false); err != nil {
+	if err = er.deleteObjectVersion(ctx, bucket, object, fi, false); err != nil {
 		eventName = event.ObjectTransitionFailed
 	}
 

cmd/erasure-server-pool-decom.go

@@ -800,11 +800,14 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						logger.LogIf(ctx, err)
 						continue
 					}
+					stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
+						stopFn(err)
 						failure = true
 						logger.LogIf(ctx, err)
 						continue
 					}
+					stopFn(nil)
 					failure = false
 					break
 				}
@@ -819,6 +822,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 			// if all versions were decommissioned, then we can delete the object versions.
 			if decommissionedCount == len(fivs.Versions) {
+				stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name)
 				_, err := set.DeleteObject(ctx,
 					bi.Name,
 					encodeDirObject(entry.name),
@@ -826,6 +830,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						DeletePrefix: true, // use prefix delete to delete all versions at once.
 					},
 				)
+				stopFn(err)
 				auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err)
 				if err != nil {
 					logger.LogIf(ctx, err)
@@ -881,6 +886,47 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 	return nil
 }
 
+//msgp:ignore decomMetrics
+type decomMetrics struct{}
+
+var globalDecommissionMetrics decomMetrics
+
+//msgp:ignore decomMetric
+//go:generate stringer -type=decomMetric -trimprefix=decomMetric $GOFILE
+type decomMetric uint8
+
+const (
+	decomMetricDecommissionBucket decomMetric = iota
+	decomMetricDecommissionObject
+	decomMetricDecommissionRemoveObject
+)
+
+func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo {
+	var errStr string
+	if err != nil {
+		errStr = err.Error()
+	}
+	return madmin.TraceInfo{
+		TraceType: madmin.TraceDecommission,
+		Time:      startTime,
+		NodeName:  globalLocalNodeName,
+		FuncName:  fmt.Sprintf("decommission.%s (pool-id=%d)", d.String(), poolIdx),
+		Duration:  duration,
+		Path:      path,
+		Error:     errStr,
+	}
+}
+
+func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err error) {
+	startTime := time.Now()
+	return func(err error) {
+		duration := time.Since(startTime)
+		if globalTrace.NumSubscribers(madmin.TraceDecommission) > 0 {
+			globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err))
+		}
+	}
+}
+
 func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
 	pool := z.serverPools[idx]
 	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
@@ -898,9 +944,13 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 		if serverDebugLog {
 			console.Debugln("decommission: currently on bucket", bucket.Name)
 		}
+		stopFn := globalDecommissionMetrics.log(decomMetricDecommissionBucket, idx, bucket.Name)
 		if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil {
+			stopFn(err)
 			return err
 		}
+		stopFn(nil)
+
 		z.poolMetaMutex.Lock()
 		z.poolMeta.BucketDone(idx, bucket)
 		z.poolMeta.save(ctx, z.serverPools)

cmd/erasure-server-pool.go

@@ -380,19 +380,15 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 	return serverPools
 }
 
-// poolObjInfo represents the state of an object per pool
-type poolObjInfo struct {
-	PoolIndex int
-	ObjInfo   ObjectInfo
-	Err       error
+// PoolObjInfo represents the state of current object version per pool
+type PoolObjInfo struct {
+	Index   int
+	ObjInfo ObjectInfo
+	Err     error
 }
 
-func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (idx int, err error) {
-	if z.SinglePool() {
-		return 0, nil
-	}
-
-	poolObjInfos := make([]poolObjInfo, len(z.serverPools))
+func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (PoolObjInfo, error) {
+	poolObjInfos := make([]PoolObjInfo, len(z.serverPools))
 	poolOpts := make([]ObjectOptions, len(z.serverPools))
 	for i := range z.serverPools {
 		poolOpts[i] = opts
@@ -404,8 +400,8 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 		go func(i int, pool *erasureSets, opts ObjectOptions) {
 			defer wg.Done()
 			// remember the pool index, we may sort the slice original index might be lost.
-			pinfo := poolObjInfo{
-				PoolIndex: i,
+			pinfo := PoolObjInfo{
+				Index: i,
 			}
 			// do not remove this check as it can lead to inconsistencies
 			// for all callers of bucket replication.
@@ -429,19 +425,19 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 	for _, pinfo := range poolObjInfos {
 		// skip all objects from suspended pools if asked by the
 		// caller.
-		if z.IsSuspended(pinfo.PoolIndex) && opts.SkipDecommissioned {
+		if z.IsSuspended(pinfo.Index) && opts.SkipDecommissioned {
 			continue
 		}
 
 		if pinfo.Err != nil && !isErrObjectNotFound(pinfo.Err) {
-			return -1, pinfo.Err
+			return pinfo, pinfo.Err
 		}
 
 		if isErrObjectNotFound(pinfo.Err) {
 			// No object exists or its a delete marker,
 			// check objInfo to confirm.
 			if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" {
-				return pinfo.PoolIndex, nil
+				return pinfo, nil
 			}
 
 			// objInfo is not valid, truly the object doesn't
@@ -449,10 +445,18 @@ func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, buc
 			continue
 		}
 
-		return pinfo.PoolIndex, nil
+		return pinfo, nil
 	}
 
-	return -1, toObjectErr(errFileNotFound, bucket, object)
+	return PoolObjInfo{}, toObjectErr(errFileNotFound, bucket, object)
+}
+
+func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (idx int, err error) {
+	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, opts)
+	if err != nil {
+		return -1, err
+	}
+	return pinfo.Index, nil
 }
 
 // getPoolIdxExistingNoLock returns the (first) found object pool index containing an object.
@@ -999,16 +1003,36 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	}
 
 	object = encodeDirObject(object)
-	if z.SinglePool() {
-		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
-	}
 
-	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
+	// Acquire a write lock before deleting the object.
+	lk := z.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout)
 	if err != nil {
+		return ObjectInfo{}, err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx.Cancel)
+
+	gopts := opts
+	gopts.NoLock = true
+	pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts)
+	if err != nil {
+		switch err.(type) {
+		case InsufficientReadQuorum:
+			return objInfo, InsufficientWriteQuorum{}
+		}
 		return objInfo, err
 	}
 
-	return z.serverPools[idx].DeleteObject(ctx, bucket, object, opts)
+	// Delete marker already present we are not going to create new delete markers.
+	if pinfo.ObjInfo.DeleteMarker && opts.VersionID == "" {
+		pinfo.ObjInfo.Name = decodeDirObject(object)
+		return pinfo.ObjInfo, nil
+	}
+
+	objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts)
+	objInfo.Name = decodeDirObject(object)
+	return objInfo, err
 }
 
 func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
@@ -1034,64 +1058,84 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 	ctx = lkctx.Context()
 	defer multiDeleteLock.Unlock(lkctx.Cancel)
 
-	if z.SinglePool() {
-		deleteObjects, dErrs := z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
-		for i := range deleteObjects {
-			deleteObjects[i].ObjectName = decodeDirObject(deleteObjects[i].ObjectName)
-		}
-		return deleteObjects, dErrs
-	}
-
 	// Fetch location of up to 10 objects concurrently.
 	poolObjIdxMap := map[int][]ObjectToDelete{}
 	origIndexMap := map[int][]int{}
 
+	// Always perform 1/10th of the number of objects per delete
+	concurrent := len(objects) / 10
+	if concurrent <= 10 {
+		// if we cannot get 1/10th then choose the number of
+		// objects as concurrent.
+		concurrent = len(objects)
+	}
+
 	var mu sync.Mutex
-	eg := errgroup.WithNErrs(len(objects)).WithConcurrency(10)
+	eg := errgroup.WithNErrs(len(objects)).WithConcurrency(concurrent)
 	for j, obj := range objects {
 		j := j
 		obj := obj
 		eg.Go(func() error {
-			idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, obj.ObjectName, ObjectOptions{
+			pinfo, err := z.getPoolInfoExistingWithOpts(ctx, bucket, obj.ObjectName, ObjectOptions{
 				NoLock: true,
 			})
 			if err != nil {
 				derrs[j] = err
+				dobjects[j] = DeletedObject{
+					ObjectName: obj.ObjectName,
+				}
 				return nil
 			}
+
+			// Delete marker already present we are not going to create new delete markers.
+			if pinfo.ObjInfo.DeleteMarker && obj.VersionID == "" {
+				dobjects[j] = DeletedObject{
+					DeleteMarker:          pinfo.ObjInfo.DeleteMarker,
+					DeleteMarkerVersionID: pinfo.ObjInfo.VersionID,
+					DeleteMarkerMTime:     DeleteMarkerMTime{pinfo.ObjInfo.ModTime},
+					ObjectName:            pinfo.ObjInfo.Name,
+				}
+				return nil
+			}
+
+			idx := pinfo.Index
+
 			mu.Lock()
+			defer mu.Unlock()
+
 			poolObjIdxMap[idx] = append(poolObjIdxMap[idx], obj)
 			origIndexMap[idx] = append(origIndexMap[idx], j)
-			mu.Unlock()
 			return nil
 		}, j)
 	}
 
 	eg.Wait() // wait to check all the pools.
 
-	// Delete concurrently in all server pools.
-	var wg sync.WaitGroup
-	wg.Add(len(z.serverPools))
-	for idx, pool := range z.serverPools {
-		go func(idx int, pool *erasureSets) {
-			defer wg.Done()
-			objs := poolObjIdxMap[idx]
-			if len(objs) > 0 {
-				orgIndexes := origIndexMap[idx]
-				deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
-				mu.Lock()
-				for i, derr := range errs {
-					if derr != nil {
-						derrs[orgIndexes[i]] = derr
+	if len(poolObjIdxMap) > 0 {
+		// Delete concurrently in all server pools.
+		var wg sync.WaitGroup
+		wg.Add(len(z.serverPools))
+		for idx, pool := range z.serverPools {
+			go func(idx int, pool *erasureSets) {
+				defer wg.Done()
+				objs := poolObjIdxMap[idx]
+				if len(objs) > 0 {
+					orgIndexes := origIndexMap[idx]
+					deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
+					mu.Lock()
+					for i, derr := range errs {
+						if derr != nil {
+							derrs[orgIndexes[i]] = derr
+						}
+						deletedObjects[i].ObjectName = decodeDirObject(deletedObjects[i].ObjectName)
+						dobjects[orgIndexes[i]] = deletedObjects[i]
 					}
-					deletedObjects[i].ObjectName = decodeDirObject(deletedObjects[i].ObjectName)
-					dobjects[orgIndexes[i]] = deletedObjects[i]
+					mu.Unlock()
 				}
-				mu.Unlock()
-			}
-		}(idx, pool)
+			}(idx, pool)
+		}
+		wg.Wait()
 	}
-	wg.Wait()
 
 	return dobjects, derrs
 }
@@ -1178,7 +1222,6 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	if marker == "" && versionMarker != "" {
 		return loi, NotImplemented{}
 	}
-
 	opts := listPathOptions{
 		Bucket:      bucket,
 		Prefix:      prefix,
@@ -1189,6 +1232,8 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		AskDisks:    globalAPIConfig.getListQuorum(),
 		Versioned:   true,
 	}
+	// set bucket metadata in opts
+	opts.setBucketMeta(ctx)
 
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
@@ -1235,12 +1280,16 @@ func maxKeysPlusOne(maxKeys int, addOne bool) int {
 
 func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	var loi ListObjectsInfo
-
-	// Automatically remove the object/version is an expiry lifecycle rule can be applied
-	lc, _ := globalLifecycleSys.Get(bucket)
-
-	// Check if bucket is object locked.
-	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
+	opts := listPathOptions{
+		Bucket:      bucket,
+		Prefix:      prefix,
+		Separator:   delimiter,
+		Limit:       maxKeysPlusOne(maxKeys, marker != ""),
+		Marker:      marker,
+		InclDeleted: false,
+		AskDisks:    globalAPIConfig.getListQuorum(),
+	}
+	opts.setBucketMeta(ctx)
 
 	if len(prefix) > 0 && maxKeys == 1 && delimiter == "" && marker == "" {
 		// Optimization for certain applications like
@@ -1252,8 +1301,8 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		// to avoid the need for ListObjects().
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
-			if lc != nil {
-				action := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo, false)
+			if opts.Lifecycle != nil {
+				action := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo, false)
 				switch action {
 				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
 					fallthrough
@@ -1266,18 +1315,6 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		}
 	}
 
-	opts := listPathOptions{
-		Bucket:      bucket,
-		Prefix:      prefix,
-		Separator:   delimiter,
-		Limit:       maxKeysPlusOne(maxKeys, marker != ""),
-		Marker:      marker,
-		InclDeleted: false,
-		AskDisks:    globalAPIConfig.getListQuorum(),
-		Lifecycle:   lc,
-		Retention:   rcfg,
-	}
-
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
@@ -1834,18 +1871,17 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 							cancel()
 							return
 						}
-						if opts.WalkAscending {
-							for i := len(fivs.Versions) - 1; i >= 0; i-- {
-								version := fivs.Versions[i]
-								versioned := vcfg != nil && vcfg.Versioned(version.Name)
 
-								results <- version.ToObjectInfo(bucket, version.Name, versioned)
-							}
-							return
-						}
+						versionsSorter(fivs.Versions).reverse()
+
 						for _, version := range fivs.Versions {
 							versioned := vcfg != nil && vcfg.Versioned(version.Name)
-							results <- version.ToObjectInfo(bucket, version.Name, versioned)
+
+							select {
+							case <-ctx.Done():
+								return
+							case results <- version.ToObjectInfo(bucket, version.Name, versioned):
+							}
 						}
 					}
 
@@ -1887,6 +1923,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 
 					if err := listPathRaw(ctx, lopts); err != nil {
 						logger.LogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
+						cancel()
 						return
 					}
 				}()

cmd/erasure-single-drive.go

@@ -1223,7 +1223,7 @@ func (es *erasureSingle) putObject(ctx context.Context, bucket string, object st
 	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
-func (es *erasureSingle) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo, forceDelMarker bool) error {
+func (es *erasureSingle) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker bool) error {
 	return es.disk.DeleteVersion(ctx, bucket, object, fi, forceDelMarker)
 }
 
@@ -1447,7 +1447,7 @@ func (es *erasureSingle) DeleteObject(ctx context.Context, bucket, object string
 
 	versionFound := true
 	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
-	goi, writeQuorum, gerr := es.getObjectInfoAndQuorum(ctx, bucket, object, opts)
+	goi, _, gerr := es.getObjectInfoAndQuorum(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
 		switch gerr.(type) {
 		case InsufficientReadQuorum:
@@ -1461,6 +1461,11 @@ func (es *erasureSingle) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
+	// Do not create new delete markers.
+	if goi.DeleteMarker && opts.VersionID == "" {
+		return goi, nil
+	}
+
 	if opts.Expiration.Expire {
 		action := evalActionFromLifecycle(ctx, *lc, rcfg, goi, false)
 		var isErr bool
@@ -1557,7 +1562,7 @@ func (es *erasureSingle) DeleteObject(ctx context.Context, bucket, object string
 			// delete marker. Add delete marker, since we don't have
 			// any version specified explicitly. Or if a particular
 			// version id needs to be replicated.
-			if err = es.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
+			if err = es.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
 				return objInfo, toObjectErr(err, bucket, object)
 			}
 			return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
@@ -1576,7 +1581,7 @@ func (es *erasureSingle) DeleteObject(ctx context.Context, bucket, object string
 		ExpireRestored:   opts.Transition.ExpireRestored,
 	}
 	dfi.SetTierFreeVersionID(fvID)
-	if err = es.deleteObjectVersion(ctx, bucket, object, writeQuorum, dfi, opts.DeleteMarker); err != nil {
+	if err = es.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 
@@ -1813,14 +1818,7 @@ func (es *erasureSingle) TransitionObject(ctx context.Context, bucket, object st
 	fi.TransitionVersionID = string(rv)
 	eventName := event.ObjectTransitionComplete
 
-	// we now know the number of blocks this object needs for data and parity.
-	// writeQuorum is dataBlocks + 1
-	writeQuorum := fi.Erasure.DataBlocks
-	if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks {
-		writeQuorum++
-	}
-
-	if err = es.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, false); err != nil {
+	if err = es.deleteObjectVersion(ctx, bucket, object, fi, false); err != nil {
 		eventName = event.ObjectTransitionFailed
 	}
 
@@ -2859,11 +2857,16 @@ func (es *erasureSingle) AbortMultipartUpload(ctx context.Context, bucket, objec
 func (es *erasureSingle) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	var loi ListObjectsInfo
 
-	// Automatically remove the object/version is an expiry lifecycle rule can be applied
-	lc, _ := globalLifecycleSys.Get(bucket)
-
-	// Check if bucket is object locked.
-	rcfg, _ := globalBucketObjectLockSys.Get(bucket)
+	opts := listPathOptions{
+		Bucket:      bucket,
+		Prefix:      prefix,
+		Separator:   delimiter,
+		Limit:       maxKeysPlusOne(maxKeys, marker != ""),
+		Marker:      marker,
+		InclDeleted: false,
+		AskDisks:    globalAPIConfig.getListQuorum(),
+	}
+	opts.setBucketMeta(ctx)
 
 	if len(prefix) > 0 && maxKeys == 1 && delimiter == "" && marker == "" {
 		// Optimization for certain applications like
@@ -2875,8 +2878,8 @@ func (es *erasureSingle) ListObjects(ctx context.Context, bucket, prefix, marker
 		// to avoid the need for ListObjects().
 		objInfo, err := es.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
-			if lc != nil {
-				action := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo, false)
+			if opts.Lifecycle != nil {
+				action := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo, false)
 				switch action {
 				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
 					fallthrough
@@ -2889,18 +2892,6 @@ func (es *erasureSingle) ListObjects(ctx context.Context, bucket, prefix, marker
 		}
 	}
 
-	opts := listPathOptions{
-		Bucket:      bucket,
-		Prefix:      prefix,
-		Separator:   delimiter,
-		Limit:       maxKeysPlusOne(maxKeys, marker != ""),
-		Marker:      marker,
-		InclDeleted: false,
-		AskDisks:    globalAPIConfig.getListQuorum(),
-		Lifecycle:   lc,
-		Retention:   rcfg,
-	}
-
 	merged, err := es.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
@@ -2959,7 +2950,6 @@ func (es *erasureSingle) ListObjectVersions(ctx context.Context, bucket, prefix,
 	if marker == "" && versionMarker != "" {
 		return loi, NotImplemented{}
 	}
-
 	opts := listPathOptions{
 		Bucket:      bucket,
 		Prefix:      prefix,
@@ -2970,6 +2960,7 @@ func (es *erasureSingle) ListObjectVersions(ctx context.Context, bucket, prefix,
 		AskDisks:    "strict",
 		Versioned:   true,
 	}
+	opts.setBucketMeta(ctx)
 
 	merged, err := es.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
@@ -3016,13 +3007,13 @@ func (es *erasureSingle) Walk(ctx context.Context, bucket, prefix string, result
 		return err
 	}
 
+	vcfg, _ := globalBucketVersioningSys.Get(bucket)
+
 	ctx, cancel := context.WithCancel(ctx)
 	go func() {
 		defer cancel()
 		defer close(results)
 
-		versioned := opts.Versioned || opts.VersionSuspended
-
 		var wg sync.WaitGroup
 		wg.Add(1)
 		go func() {
@@ -3037,15 +3028,17 @@ func (es *erasureSingle) Walk(ctx context.Context, bucket, prefix string, result
 					cancel()
 					return
 				}
-				if opts.WalkAscending {
-					for i := len(fivs.Versions) - 1; i >= 0; i-- {
-						version := fivs.Versions[i]
-						results <- version.ToObjectInfo(bucket, version.Name, versioned)
-					}
-					return
-				}
+
+				versionsSorter(fivs.Versions).reverse()
+
 				for _, version := range fivs.Versions {
-					results <- version.ToObjectInfo(bucket, version.Name, versioned)
+					versioned := vcfg != nil && vcfg.Versioned(version.Name)
+
+					select {
+					case <-ctx.Done():
+						return
+					case results <- version.ToObjectInfo(bucket, version.Name, versioned):
+					}
 				}
 			}
 
@@ -3087,6 +3080,7 @@ func (es *erasureSingle) Walk(ctx context.Context, bucket, prefix string, result
 
 			if err := listPathRaw(ctx, lopts); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
+				cancel()
 				return
 			}
 		}()

cmd/erasure.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"runtime"
 	"sort"
 	"sync"
 	"time"
@@ -200,8 +201,11 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				UsedSpace:      info.Used,
 				AvailableSpace: info.Free,
 				UUID:           info.ID,
+				Major:          info.Major,
+				Minor:          info.Minor,
 				RootDisk:       info.RootDisk,
 				Healing:        info.Healing,
+				Scanning:       info.Scanning,
 				State:          diskErrToDriveState(err),
 				FreeInodes:     info.FreeInodes,
 			}
@@ -439,6 +443,13 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	r.Shuffle(len(disks), func(i, j int) { disks[i], disks[j] = disks[j], disks[i] })
 
+	// Restrict parallelism for disk usage scanner
+	// upto GOMAXPROCS if GOMAXPROCS is < len(disks)
+	maxProcs := runtime.GOMAXPROCS(0)
+	if maxProcs < len(disks) {
+		disks = disks[:maxProcs]
+	}
+
 	// Start one scanner per disk
 	var wg sync.WaitGroup
 	wg.Add(len(disks))

cmd/event-notification.go

@@ -0,0 +1,323 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"strings"
+	"sync"
+
+	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/event"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
+)
+
+// EventNotifier - notifies external systems about events in MinIO.
+type EventNotifier struct {
+	sync.RWMutex
+	targetList                 *event.TargetList
+	targetResCh                chan event.TargetIDResult
+	bucketRulesMap             map[string]event.RulesMap
+	bucketRemoteTargetRulesMap map[string]map[event.TargetID]event.RulesMap
+}
+
+// NewEventNotifier - creates new event notification object.
+func NewEventNotifier() *EventNotifier {
+	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.Init()
+	return &EventNotifier{
+		targetList:                 event.NewTargetList(),
+		targetResCh:                make(chan event.TargetIDResult),
+		bucketRulesMap:             make(map[string]event.RulesMap),
+		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
+	}
+}
+
+// GetARNList - returns available ARNs.
+func (evnot *EventNotifier) GetARNList(onlyActive bool) []string {
+	arns := []string{}
+	if evnot == nil {
+		return arns
+	}
+	region := globalSite.Region
+	for targetID, target := range evnot.targetList.TargetMap() {
+		// httpclient target is part of ListenNotification
+		// which doesn't need to be listed as part of the ARN list
+		// This list is only meant for external targets, filter
+		// this out pro-actively.
+		if !strings.HasPrefix(targetID.ID, "httpclient+") {
+			if onlyActive && !target.HasQueueStore() {
+				if _, err := target.IsActive(); err != nil {
+					continue
+				}
+			}
+			arns = append(arns, targetID.ToARN(region).String())
+		}
+	}
+
+	return arns
+}
+
+// Loads notification policies for all buckets into EventNotifier.
+func (evnot *EventNotifier) set(bucket BucketInfo, meta BucketMetadata) {
+	config := meta.notificationConfig
+	if config == nil {
+		return
+	}
+	config.SetRegion(globalSite.Region)
+	if err := config.Validate(globalSite.Region, globalEventNotifier.targetList); err != nil {
+		if _, ok := err.(*event.ErrARNNotFound); !ok {
+			logger.LogIf(GlobalContext, err)
+		}
+	}
+	evnot.AddRulesMap(bucket.Name, config.ToRulesMap())
+}
+
+// InitBucketTargets - initializes notification system from notification.xml of all buckets.
+func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI ObjectLayer) error {
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// In gateway mode, notifications are not supported - except NAS gateway.
+	if globalIsGateway && !objAPI.IsNotificationSupported() {
+		return nil
+	}
+
+	logger.LogIf(ctx, evnot.targetList.Add(globalConfigTargetList.Targets()...))
+
+	go func() {
+		for res := range evnot.targetResCh {
+			if res.Err != nil {
+				reqInfo := &logger.ReqInfo{}
+				reqInfo.AppendTags("targetID", res.ID.Name)
+				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
+			}
+		}
+	}()
+
+	return nil
+}
+
+// AddRulesMap - adds rules map for bucket name.
+func (evnot *EventNotifier) AddRulesMap(bucketName string, rulesMap event.RulesMap) {
+	evnot.Lock()
+	defer evnot.Unlock()
+
+	rulesMap = rulesMap.Clone()
+
+	for _, targetRulesMap := range evnot.bucketRemoteTargetRulesMap[bucketName] {
+		rulesMap.Add(targetRulesMap)
+	}
+
+	// Do not add for an empty rulesMap.
+	if len(rulesMap) == 0 {
+		delete(evnot.bucketRulesMap, bucketName)
+	} else {
+		evnot.bucketRulesMap[bucketName] = rulesMap
+	}
+}
+
+// RemoveRulesMap - removes rules map for bucket name.
+func (evnot *EventNotifier) RemoveRulesMap(bucketName string, rulesMap event.RulesMap) {
+	evnot.Lock()
+	defer evnot.Unlock()
+
+	evnot.bucketRulesMap[bucketName].Remove(rulesMap)
+	if len(evnot.bucketRulesMap[bucketName]) == 0 {
+		delete(evnot.bucketRulesMap, bucketName)
+	}
+}
+
+// ConfiguredTargetIDs - returns list of configured target id's
+func (evnot *EventNotifier) ConfiguredTargetIDs() []event.TargetID {
+	if evnot == nil {
+		return nil
+	}
+
+	evnot.RLock()
+	defer evnot.RUnlock()
+
+	var targetIDs []event.TargetID
+	for _, rmap := range evnot.bucketRulesMap {
+		for _, rules := range rmap {
+			for _, targetSet := range rules {
+				for id := range targetSet {
+					targetIDs = append(targetIDs, id)
+				}
+			}
+		}
+	}
+	// Filter out targets configured via env
+	var tIDs []event.TargetID
+	for _, targetID := range targetIDs {
+		if !globalEnvTargetList.Exists(targetID) {
+			tIDs = append(tIDs, targetID)
+		}
+	}
+	return tIDs
+}
+
+// RemoveNotification - removes all notification configuration for bucket name.
+func (evnot *EventNotifier) RemoveNotification(bucketName string) {
+	evnot.Lock()
+	defer evnot.Unlock()
+
+	delete(evnot.bucketRulesMap, bucketName)
+
+	targetIDSet := event.NewTargetIDSet()
+	for targetID := range evnot.bucketRemoteTargetRulesMap[bucketName] {
+		targetIDSet[targetID] = struct{}{}
+		delete(evnot.bucketRemoteTargetRulesMap[bucketName], targetID)
+	}
+	evnot.targetList.Remove(targetIDSet)
+
+	delete(evnot.bucketRemoteTargetRulesMap, bucketName)
+}
+
+// RemoveAllRemoteTargets - closes and removes all notification targets.
+func (evnot *EventNotifier) RemoveAllRemoteTargets() {
+	evnot.Lock()
+	defer evnot.Unlock()
+
+	for _, targetMap := range evnot.bucketRemoteTargetRulesMap {
+		targetIDSet := event.NewTargetIDSet()
+		for k := range targetMap {
+			targetIDSet[k] = struct{}{}
+		}
+		evnot.targetList.Remove(targetIDSet)
+	}
+}
+
+// Send - sends event data to all matching targets.
+func (evnot *EventNotifier) Send(args eventArgs) {
+	evnot.RLock()
+	targetIDSet := evnot.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
+	evnot.RUnlock()
+
+	if len(targetIDSet) == 0 {
+		return
+	}
+
+	evnot.targetList.Send(args.ToEvent(true), targetIDSet, evnot.targetResCh)
+}
+
+type eventArgs struct {
+	EventName    event.Name
+	BucketName   string
+	Object       ObjectInfo
+	ReqParams    map[string]string
+	RespElements map[string]string
+	Host         string
+	UserAgent    string
+}
+
+// ToEvent - converts to notification event.
+func (args eventArgs) ToEvent(escape bool) event.Event {
+	eventTime := UTCNow()
+	uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
+
+	respElements := map[string]string{
+		"x-amz-request-id": args.RespElements["requestId"],
+		"x-minio-origin-endpoint": func() string {
+			if globalMinioEndpoint != "" {
+				return globalMinioEndpoint
+			}
+			return getAPIEndpoints()[0]
+		}(), // MinIO specific custom elements.
+	}
+	// Add deployment as part of
+	if globalDeploymentID != "" {
+		respElements["x-minio-deployment-id"] = globalDeploymentID
+	}
+	if args.RespElements["content-length"] != "" {
+		respElements["content-length"] = args.RespElements["content-length"]
+	}
+	keyName := args.Object.Name
+	if escape {
+		keyName = url.QueryEscape(args.Object.Name)
+	}
+	newEvent := event.Event{
+		EventVersion:      "2.0",
+		EventSource:       "minio:s3",
+		AwsRegion:         args.ReqParams["region"],
+		EventTime:         eventTime.Format(event.AMZTimeFormat),
+		EventName:         args.EventName,
+		UserIdentity:      event.Identity{PrincipalID: args.ReqParams["principalId"]},
+		RequestParameters: args.ReqParams,
+		ResponseElements:  respElements,
+		S3: event.Metadata{
+			SchemaVersion:   "1.0",
+			ConfigurationID: "Config",
+			Bucket: event.Bucket{
+				Name:          args.BucketName,
+				OwnerIdentity: event.Identity{PrincipalID: args.ReqParams["principalId"]},
+				ARN:           policy.ResourceARNPrefix + args.BucketName,
+			},
+			Object: event.Object{
+				Key:       keyName,
+				VersionID: args.Object.VersionID,
+				Sequencer: uniqueID,
+			},
+		},
+		Source: event.Source{
+			Host:      args.Host,
+			UserAgent: args.UserAgent,
+		},
+	}
+
+	if args.EventName != event.ObjectRemovedDelete && args.EventName != event.ObjectRemovedDeleteMarkerCreated {
+		newEvent.S3.Object.ETag = args.Object.ETag
+		newEvent.S3.Object.Size = args.Object.Size
+		newEvent.S3.Object.ContentType = args.Object.ContentType
+		newEvent.S3.Object.UserMetadata = make(map[string]string, len(args.Object.UserDefined))
+		for k, v := range args.Object.UserDefined {
+			if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+				continue
+			}
+			newEvent.S3.Object.UserMetadata[k] = v
+		}
+	}
+
+	return newEvent
+}
+
+func sendEvent(args eventArgs) {
+	args.Object.Size, _ = args.Object.GetActualSize()
+
+	// avoid generating a notification for REPLICA creation event.
+	if _, ok := args.ReqParams[xhttp.MinIOSourceReplicationRequest]; ok {
+		return
+	}
+	// remove sensitive encryption entries in metadata.
+	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
+	crypto.RemoveInternalEntries(args.Object.UserDefined)
+
+	// globalNotificationSys is not initialized in gateway mode.
+	if globalNotificationSys == nil {
+		return
+	}
+	if globalHTTPListen.NumSubscribers(args.EventName) > 0 {
+		globalHTTPListen.Publish(args.ToEvent(false))
+	}
+
+	globalEventNotifier.Send(args)
+}

cmd/gateway-main.go

@@ -316,7 +316,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		}
 		logger.FatalIf(globalBucketMetadataSys.Init(GlobalContext, buckets, newObject), "Unable to initialize bucket metadata")
 
-		logger.FatalIf(globalNotificationSys.InitBucketTargets(GlobalContext, newObject), "Unable to initialize bucket targets for notification system")
+		logger.FatalIf(globalEventNotifier.InitBucketTargets(GlobalContext, newObject), "Unable to initialize bucket targets for notification system")
 	}
 
 	if globalCacheConfig.Enabled {

cmd/globals.go

@@ -188,7 +188,9 @@ var (
 	// globalConfigSys server config system.
 	globalConfigSys *ConfigSys
 
-	globalNotificationSys  *NotificationSys
+	globalNotificationSys *NotificationSys
+
+	globalEventNotifier    *EventNotifier
 	globalConfigTargetList *event.TargetList
 	// globalEnvTargetList has list of targets configured via env.
 	globalEnvTargetList *event.TargetList
@@ -383,18 +385,30 @@ var (
 	// Add new variable global values here.
 )
 
-var globalAuthZPluginMutex sync.Mutex
+var globalAuthPluginMutex sync.Mutex
+
+func newGlobalAuthNPluginFn() *idplugin.AuthNPlugin {
+	globalAuthPluginMutex.Lock()
+	defer globalAuthPluginMutex.Unlock()
+	return globalAuthNPlugin
+}
 
 func newGlobalAuthZPluginFn() *polplugin.AuthZPlugin {
-	globalAuthZPluginMutex.Lock()
-	defer globalAuthZPluginMutex.Unlock()
+	globalAuthPluginMutex.Lock()
+	defer globalAuthPluginMutex.Unlock()
 	return globalAuthZPlugin
 }
 
+func setGlobalAuthNPlugin(authn *idplugin.AuthNPlugin) {
+	globalAuthPluginMutex.Lock()
+	globalAuthNPlugin = authn
+	globalAuthPluginMutex.Unlock()
+}
+
 func setGlobalAuthZPlugin(authz *polplugin.AuthZPlugin) {
-	globalAuthZPluginMutex.Lock()
+	globalAuthPluginMutex.Lock()
 	globalAuthZPlugin = authz
-	globalAuthZPluginMutex.Unlock()
+	globalAuthPluginMutex.Unlock()
 }
 
 var errSelfTestFailure = errors.New("self test failed. unsafe to start server")

cmd/iam-store.go

@@ -862,7 +862,10 @@ func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err err
 }
 
 // PolicyDBSet - update the policy mapping for the given user or group in
-// storage and in cache.
+// storage and in cache. We do not check for the existence of the user here
+// since users can be virtual, such as for:
+//   - LDAP users
+//   - CommonName for STS accounts generated by AssumeRoleWithCertificate
 func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) (updatedAt time.Time, err error) {
 	if name == "" {
 		return updatedAt, errInvalidArgument
@@ -871,19 +874,6 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 	cache := store.lock()
 	defer store.unlock()
 
-	// Validate that user and group exist.
-	if store.getUsersSysType() == MinIOUsersSysType {
-		if !isGroup {
-			if _, ok := cache.iamUsersMap[name]; !ok {
-				return updatedAt, errNoSuchUser
-			}
-		} else {
-			if _, ok := cache.iamGroupsMap[name]; !ok {
-				return updatedAt, errNoSuchGroup
-			}
-		}
-	}
-
 	// Handle policy mapping removal.
 	if policy == "" {
 		if store.getUsersSysType() == LDAPUsersSysType {

cmd/iam.go

@@ -38,8 +38,13 @@ import (
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/config"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
+	idplugin "github.com/minio/minio/internal/config/identity/plugin"
+	"github.com/minio/minio/internal/config/policy/opa"
+	polplugin "github.com/minio/minio/internal/config/policy/plugin"
+	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -100,7 +105,8 @@ type IAMSys struct {
 type IAMUserType int
 
 const (
-	regUser IAMUserType = iota
+	unknownIAMUserType IAMUserType = iota - 1
+	regUser
 	stsUser
 	svcUser
 )
@@ -129,17 +135,11 @@ func (sys *IAMSys) LoadPolicy(ctx context.Context, objAPI ObjectLayer, policyNam
 
 // LoadPolicyMapping - loads the mapped policy for a user or group
 // from storage into server memory.
-func (sys *IAMSys) LoadPolicyMapping(ctx context.Context, objAPI ObjectLayer, userOrGroup string, isGroup bool) error {
+func (sys *IAMSys) LoadPolicyMapping(ctx context.Context, objAPI ObjectLayer, userOrGroup string, userType IAMUserType, isGroup bool) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
-	// In case of LDAP, policy mappings are only applicable to sts users.
-	userType := regUser
-	if sys.usersSysType == LDAPUsersSysType {
-		userType = stsUser
-	}
-
 	return sys.store.PolicyMappingNotificationHandler(ctx, userOrGroup, isGroup, userType)
 }
 
@@ -164,7 +164,7 @@ func (sys *IAMSys) LoadServiceAccount(ctx context.Context, accessKey string) err
 // initStore initializes IAM stores
 func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
 	if sys.ldapConfig.Enabled {
-		sys.EnableLDAPSys()
+		sys.SetUsersSysType(LDAPUsersSysType)
 	}
 
 	if etcdClient == nil {
@@ -218,6 +218,54 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
+	globalServerConfigMu.RLock()
+	s := globalServerConfig
+	globalServerConfigMu.RUnlock()
+
+	ldapCfg := s[config.IdentityLDAPSubSys][config.Default]
+
+	var err error
+	globalOpenIDConfig, err = openid.LookupConfig(s,
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
+	}
+
+	// Initialize if LDAP is enabled
+	globalLDAPConfig, err = xldap.Lookup(ldapCfg, globalRootCAs)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
+	}
+
+	authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default],
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err))
+	}
+
+	setGlobalAuthNPlugin(idplugin.New(authNPluginCfg))
+
+	authZPluginCfg, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
+		NewGatewayHTTPTransport(), xhttp.DrainBody)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err))
+	}
+
+	if authZPluginCfg.URL == nil {
+		opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default],
+			NewGatewayHTTPTransport(), xhttp.DrainBody)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err))
+		} else {
+			authZPluginCfg.URL = opaCfg.URL
+			authZPluginCfg.AuthToken = opaCfg.AuthToken
+			authZPluginCfg.Transport = opaCfg.Transport
+			authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn
+		}
+	}
+
+	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
+
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -330,8 +378,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	}
 
 	// From AuthN plugin if enabled.
-	if globalAuthNPlugin != nil {
-		riMap := globalAuthNPlugin.GetRoleInfo()
+	if authn := newGlobalAuthNPluginFn(); authn != nil {
+		riMap := authn.GetRoleInfo()
 		sys.validateAndAddRolePolicyMappings(ctx, riMap)
 	}
 
@@ -352,7 +400,8 @@ func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[a
 		knownPoliciesSet := newMappedPolicy(validPolicies).policySet()
 		unknownPoliciesSet := specifiedPoliciesSet.Difference(knownPoliciesSet)
 		if len(unknownPoliciesSet) > 0 {
-			if globalAuthZPlugin == nil {
+			authz := newGlobalAuthZPluginFn()
+			if authz == nil {
 				// Print a warning that some policies mapped to a role are not defined.
 				errMsg := fmt.Errorf(
 					"The policies \"%s\" mapped to role ARN %s are not defined - this role may not work as expected.",
@@ -1383,18 +1432,12 @@ func (sys *IAMSys) ListGroups(ctx context.Context) (r []string, err error) {
 	}
 }
 
-// PolicyDBSet - sets a policy for a user or group in the PolicyDB.
-func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup bool) (updatedAt time.Time, err error) {
+// PolicyDBSet - sets a policy for a user or group in the PolicyDB - the user doesn't have to exist since sometimes they are virtuals
+func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) (updatedAt time.Time, err error) {
 	if !sys.Initialized() {
 		return updatedAt, errServerNotInitialized
 	}
 
-	// Determine user-type based on IDP mode.
-	userType := regUser
-	if sys.usersSysType == LDAPUsersSysType {
-		userType = stsUser
-	}
-
 	updatedAt, err = sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
 	if err != nil {
 		return
@@ -1402,7 +1445,7 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup
 
 	// Notify all other MinIO peers to reload policy
 	if !sys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadPolicyMapping(name, isGroup) {
+		for _, nerr := range globalNotificationSys.LoadPolicyMapping(name, userType, isGroup) {
 			if nerr.Err != nil {
 				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
 				logger.LogIf(ctx, nerr.Err)
@@ -1735,9 +1778,14 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	return sys.GetCombinedPolicy(policies...).IsAllowed(args)
 }
 
-// EnableLDAPSys - enable ldap system users type.
-func (sys *IAMSys) EnableLDAPSys() {
-	sys.usersSysType = LDAPUsersSysType
+// SetUsersSysType - sets the users system type, regular or LDAP.
+func (sys *IAMSys) SetUsersSysType(t UsersSysType) {
+	sys.usersSysType = t
+}
+
+// GetUsersSysType - returns the users system type for this IAM
+func (sys *IAMSys) GetUsersSysType() UsersSysType {
+	return sys.usersSysType
 }
 
 // NewIAMSys - creates new config system object.

cmd/metacache-server-pool.go

@@ -29,7 +29,6 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -113,14 +112,14 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	if o.ID != "" && !o.Transient {
 		// Create or ping with handout...
 		rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
-		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-		defer cancel()
 		var c *metacache
 		if rpc == nil {
 			resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o)
 			c = &resp
 		} else {
-			c, err = rpc.GetMetacacheListing(ctx, *o)
+			rctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+			c, err = rpc.GetMetacacheListing(rctx, *o)
+			cancel()
 		}
 		if err != nil {
 			if errors.Is(err, context.Canceled) {
@@ -319,58 +318,32 @@ func (es *erasureSingle) listPath(ctx context.Context, o *listPathOptions) (entr
 	//
 	// If we don't have a list id we must ask the server if it has a cache or create a new.
 	if o.ID != "" && !o.Transient {
-		// Create or ping with handout...
-		rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
-		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-		defer cancel()
-		var c *metacache
-		if rpc == nil {
-			resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o)
-			c = &resp
-		} else {
-			c, err = rpc.GetMetacacheListing(ctx, *o)
+		resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o)
+		c := &resp
+		if c.fileNotFound {
+			// No cache found, no entries found.
+			return entries, io.EOF
 		}
-		if err != nil {
-			if errors.Is(err, context.Canceled) {
-				// Context is canceled, return at once.
-				// request canceled, no entries to return
-				return entries, io.EOF
-			}
-			if !errors.Is(err, context.DeadlineExceeded) {
-				o.debugln("listPath: got error", err)
-			}
-			o.Transient = true
+		if c.status == scanStateError || c.status == scanStateNone {
+			o.ID = ""
 			o.Create = false
-			o.ID = mustGetUUID()
+			o.debugln("scan status", c.status, " - waiting a roundtrip to create")
 		} else {
-			if c.fileNotFound {
-				// No cache found, no entries found.
-				return entries, io.EOF
-			}
-			if c.status == scanStateError || c.status == scanStateNone {
-				o.ID = ""
-				o.Create = false
-				o.debugln("scan status", c.status, " - waiting a roundtrip to create")
-			} else {
-				// Continue listing
-				o.ID = c.id
-				go func(meta metacache) {
-					// Continuously update while we wait.
-					t := time.NewTicker(metacacheMaxClientWait / 10)
-					defer t.Stop()
-					select {
-					case <-ctx.Done():
-						// Request is done, stop updating.
-						return
-					case <-t.C:
-						meta.lastHandout = time.Now()
-						if rpc == nil {
-							meta, _ = localMetacacheMgr.updateCacheEntry(meta)
-						}
-						meta, _ = rpc.UpdateMetacacheListing(ctx, meta)
-					}
-				}(*c)
-			}
+			// Continue listing
+			o.ID = c.id
+			go func(meta metacache) {
+				// Continuously update while we wait.
+				t := time.NewTicker(metacacheMaxClientWait / 10)
+				defer t.Stop()
+				select {
+				case <-ctx.Done():
+					// Request is done, stop updating.
+					return
+				case <-t.C:
+					meta.lastHandout = time.Now()
+					meta, _ = localMetacacheMgr.updateCacheEntry(meta)
+				}
+			}(*c)
 		}
 
 		// We have an existing list ID, continue streaming.
@@ -400,21 +373,7 @@ func (es *erasureSingle) listPath(ctx context.Context, o *listPathOptions) (entr
 			return entries, err
 		}
 		entries.truncate(0)
-		go func() {
-			rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
-			if rpc != nil {
-				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
-				defer cancel()
-				c, err := rpc.GetMetacacheListing(ctx, *o)
-				if err == nil {
-					c.error = "no longer used"
-					c.status = scanStateError
-					rpc.UpdateMetacacheListing(ctx, *c)
-				}
-			}
-		}()
 		o.ID = ""
-
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Resuming listing from drives failed %w, proceeding to do raw listing", err))
 		}
@@ -486,9 +445,9 @@ func (es *erasureSingle) listMerged(ctx context.Context, o listPathOptions, resu
 	mu.Unlock()
 
 	// Do lifecycle filtering.
-	if o.Lifecycle != nil {
+	if o.Lifecycle != nil || o.Replication.Config != nil {
 		filterIn := make(chan metaCacheEntry, 10)
-		go filterLifeCycle(ctx, o.Bucket, *o.Lifecycle, o.Retention, filterIn, results)
+		go applyBucketActions(ctx, o, filterIn, results)
 		// Replace results.
 		results = filterIn
 	}
@@ -572,9 +531,9 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	mu.Unlock()
 
 	// Do lifecycle filtering.
-	if o.Lifecycle != nil {
+	if o.Lifecycle != nil || o.Replication.Config != nil {
 		filterIn := make(chan metaCacheEntry, 10)
-		go filterLifeCycle(ctx, o.Bucket, *o.Lifecycle, o.Retention, filterIn, results)
+		go applyBucketActions(ctx, o, filterIn, results)
 		// Replace results.
 		results = filterIn
 	}
@@ -635,15 +594,16 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	return nil
 }
 
-// filterLifeCycle will filter out objects if the most recent
-// version should be deleted by lifecycle.
+// applyBucketActions applies lifecycle and replication actions on the listing
+// It will filter out objects if the most recent version should be deleted by lifecycle.
+// Entries that failed replication will be queued if no lifecycle rules got applied.
 // out will be closed when there are no more results.
 // When 'in' is closed or the context is canceled the
 // function closes 'out' and exits.
-func filterLifeCycle(ctx context.Context, bucket string, lc lifecycle.Lifecycle, lr lock.Retention, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
+func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
 	defer close(out)
 
-	vcfg, _ := globalBucketVersioningSys.Get(bucket)
+	vcfg, _ := globalBucketVersioningSys.Get(o.Bucket)
 	for {
 		var obj metaCacheEntry
 		var ok bool
@@ -656,29 +616,32 @@ func filterLifeCycle(ctx context.Context, bucket string, lc lifecycle.Lifecycle,
 			}
 		}
 
-		fi, err := obj.fileInfo(bucket)
+		fi, err := obj.fileInfo(o.Bucket)
 		if err != nil {
 			continue
 		}
 
 		versioned := vcfg != nil && vcfg.Versioned(obj.name)
 
-		objInfo := fi.ToObjectInfo(bucket, obj.name, versioned)
-		action := evalActionFromLifecycle(ctx, lc, lr, objInfo, false)
-		switch action {
-		case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-			globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
-			// Skip this entry.
-			continue
-		case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-			globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
-			// Skip this entry.
-			continue
+		objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
+		if o.Lifecycle != nil {
+			action := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo, false)
+			switch action {
+			case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+				globalExpiryState.enqueueByDays(objInfo, false, action == lifecycle.DeleteVersionAction)
+				// Skip this entry.
+				continue
+			case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+				globalExpiryState.enqueueByDays(objInfo, true, action == lifecycle.DeleteRestoredVersionAction)
+				// Skip this entry.
+				continue
+			}
 		}
 		select {
 		case <-ctx.Done():
 			return
 		case out <- obj:
+			queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
 		}
 	}
 }

cmd/metacache-set.go

@@ -102,6 +102,8 @@ type listPathOptions struct {
 	// Retention configuration, needed to be passed along with lifecycle if set.
 	Retention lock.Retention
 
+	// Replication configuration
+	Replication replicationConfig
 	// pool and set of where the cache is located.
 	pool, set int
 }
@@ -110,6 +112,21 @@ func init() {
 	gob.Register(listPathOptions{})
 }
 
+func (o *listPathOptions) setBucketMeta(ctx context.Context) {
+	lc, _ := globalLifecycleSys.Get(o.Bucket)
+
+	// Check if bucket is object locked.
+	rcfg, _ := globalBucketObjectLockSys.Get(o.Bucket)
+	replCfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, o.Bucket)
+	tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, o.Bucket)
+	o.Lifecycle = lc
+	o.Replication = replicationConfig{
+		Config:  replCfg,
+		remotes: tgts,
+	}
+	o.Retention = rcfg
+}
+
 // newMetacache constructs a new metacache from the options.
 func (o listPathOptions) newMetacache() metacache {
 	return metacache{

cmd/metacache-stream.go

@@ -184,6 +184,9 @@ func (w *metacacheWriter) stream() (chan<- metaCacheEntry, error) {
 				continue
 			}
 			err = w.mw.WriteBytes(o.metadata)
+			if w.reuseBlocks || o.reusable {
+				metaDataPoolPut(o.metadata)
+			}
 			if err != nil {
 				w.streamErr = err
 				continue

cmd/metacache-walk.go

@@ -168,7 +168,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				entries[i] = entries[i][:len(entry)-1]
 				continue
 			}
-			// Do do not retain the file.
+			// Do not retain the file.
 			entries[i] = ""
 
 			if contextCanceled(ctx) {

cmd/metrics-realtime.go

@@ -22,24 +22,36 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/disk"
 )
 
-func collectLocalMetrics(types madmin.MetricType, hosts map[string]struct{}) (m madmin.RealtimeMetrics) {
+func collectLocalMetrics(types madmin.MetricType, hosts map[string]struct{}, disks map[string]struct{}) (m madmin.RealtimeMetrics) {
 	if types == madmin.MetricsNone {
 		return
 	}
+
 	if len(hosts) > 0 {
 		if _, ok := hosts[globalMinioAddr]; !ok {
 			return
 		}
 	}
+
+	if types.Contains(madmin.MetricsDisk) && !globalIsGateway {
+		m.ByDisk = make(map[string]madmin.DiskMetric)
+		aggr := madmin.DiskMetric{
+			CollectedAt: time.Now(),
+		}
+		for name, disk := range collectLocalDisksMetrics(disks) {
+			m.ByDisk[name] = disk
+			aggr.Merge(&disk)
+		}
+		m.Aggregated.Disk = &aggr
+	}
+
 	if types.Contains(madmin.MetricsScanner) {
 		metrics := globalScannerMetrics.report()
 		m.Aggregated.Scanner = &metrics
 	}
-	if types.Contains(madmin.MetricsDisk) && !globalIsGateway {
-		m.Aggregated.Disk = collectDiskMetrics()
-	}
 	if types.Contains(madmin.MetricsOS) {
 		metrics := globalOSMetrics.report()
 		m.Aggregated.OS = &metrics
@@ -53,55 +65,89 @@ func collectLocalMetrics(types madmin.MetricType, hosts map[string]struct{}) (m
 	return m
 }
 
-func collectDiskMetrics() *madmin.DiskMetric {
+func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskMetric {
 	objLayer := newObjectLayerFn()
-	disks := madmin.DiskMetric{
-		CollectedAt: time.Now(),
-	}
-
 	if objLayer == nil {
 		return nil
 	}
+
+	metrics := make(map[string]madmin.DiskMetric)
+
+	procStats, procErr := disk.GetAllDrivesIOStats()
+	if procErr != nil {
+		return metrics
+	}
+
 	// only need Disks information in server mode.
 	storageInfo, errs := objLayer.LocalStorageInfo(GlobalContext)
-	for _, err := range errs {
-		if err != nil {
-			disks.Merge(&madmin.DiskMetric{NDisks: 1, Offline: 1})
+
+	for i, d := range storageInfo.Disks {
+		if len(disks) != 0 {
+			_, ok := disks[d.Endpoint]
+			if !ok {
+				continue
+			}
 		}
-	}
-	for i, disk := range storageInfo.Disks {
+
 		if errs[i] != nil {
+			metrics[d.Endpoint] = madmin.DiskMetric{NDisks: 1, Offline: 1}
 			continue
 		}
-		var d madmin.DiskMetric
-		d.NDisks = 1
-		if disk.Healing {
-			d.Healing++
+
+		var dm madmin.DiskMetric
+		dm.NDisks = 1
+		if d.Healing {
+			dm.Healing++
 		}
-		if disk.Metrics != nil {
-			d.LifeTimeOps = make(map[string]uint64, len(disk.Metrics.APICalls))
-			for k, v := range disk.Metrics.APICalls {
+		if d.Metrics != nil {
+			dm.LifeTimeOps = make(map[string]uint64, len(d.Metrics.APICalls))
+			for k, v := range d.Metrics.APICalls {
 				if v != 0 {
-					d.LifeTimeOps[k] = v
+					dm.LifeTimeOps[k] = v
 				}
 			}
-			d.LastMinute.Operations = make(map[string]madmin.TimedAction, len(disk.Metrics.APICalls))
-			for k, v := range disk.Metrics.LastMinute {
+			dm.LastMinute.Operations = make(map[string]madmin.TimedAction, len(d.Metrics.APICalls))
+			for k, v := range d.Metrics.LastMinute {
 				if v.Count != 0 {
-					d.LastMinute.Operations[k] = v
+					dm.LastMinute.Operations[k] = v
 				}
 			}
 		}
-		disks.Merge(&d)
+
+		// get disk
+		if procErr == nil {
+			st := procStats[disk.DevID{Major: d.Major, Minor: d.Minor}]
+			dm.IOStats = madmin.DiskIOStats{
+				ReadIOs:        st.ReadIOs,
+				ReadMerges:     st.ReadMerges,
+				ReadSectors:    st.ReadSectors,
+				ReadTicks:      st.ReadTicks,
+				WriteIOs:       st.WriteIOs,
+				WriteMerges:    st.WriteMerges,
+				WriteSectors:   st.WriteSectors,
+				WriteTicks:     st.WriteTicks,
+				CurrentIOs:     st.CurrentIOs,
+				TotalTicks:     st.TotalTicks,
+				ReqTicks:       st.ReqTicks,
+				DiscardIOs:     st.DiscardIOs,
+				DiscardMerges:  st.DiscardMerges,
+				DiscardSectors: st.DiscardSectors,
+				DiscardTicks:   st.DiscardTicks,
+				FlushIOs:       st.FlushIOs,
+				FlushTicks:     st.FlushTicks,
+			}
+		}
+
+		metrics[d.Endpoint] = dm
 	}
-	return &disks
+	return metrics
 }
 
-func collectRemoteMetrics(ctx context.Context, types madmin.MetricType, hosts map[string]struct{}) (m madmin.RealtimeMetrics) {
+func collectRemoteMetrics(ctx context.Context, types madmin.MetricType, hosts map[string]struct{}, disks map[string]struct{}) (m madmin.RealtimeMetrics) {
 	if !globalIsDistErasure {
 		return
 	}
-	all := globalNotificationSys.GetMetrics(ctx, types, hosts)
+	all := globalNotificationSys.GetMetrics(ctx, types, hosts, disks)
 	for _, remote := range all {
 		m.Merge(&remote)
 	}

cmd/namespace-lock.go

@@ -166,7 +166,8 @@ func (di *distLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout
 
 	newCtx, cancel := context.WithCancel(ctx)
 	if !di.rwMutex.GetLock(newCtx, cancel, di.opsID, lockSource, dsync.Options{
-		Timeout: timeout.Timeout(),
+		Timeout:       timeout.Timeout(),
+		RetryInterval: timeout.RetryInterval(),
 	}) {
 		timeout.LogFailure()
 		cancel()
@@ -195,7 +196,8 @@ func (di *distLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeou
 
 	newCtx, cancel := context.WithCancel(ctx)
 	if !di.rwMutex.GetRLock(ctx, cancel, di.opsID, lockSource, dsync.Options{
-		Timeout: timeout.Timeout(),
+		Timeout:       timeout.Timeout(),
+		RetryInterval: timeout.RetryInterval(),
 	}) {
 		timeout.LogFailure()
 		cancel()

cmd/notification.go

@@ -26,7 +26,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"strings"
 	"sync"
 	"time"
 
@@ -35,49 +34,18 @@ import (
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go"
 	bucketBandwidth "github.com/minio/minio/internal/bucket/bandwidth"
-	"github.com/minio/minio/internal/crypto"
-	"github.com/minio/minio/internal/event"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
-	"github.com/minio/pkg/bucket/policy"
 	xnet "github.com/minio/pkg/net"
 )
 
+// This file contains peer related notifications. For sending notifications to
+// external systems, see event-notification.go
+
 // NotificationSys - notification system.
 type NotificationSys struct {
-	sync.RWMutex
-	targetList                 *event.TargetList
-	targetResCh                chan event.TargetIDResult
-	bucketRulesMap             map[string]event.RulesMap
-	bucketRemoteTargetRulesMap map[string]map[event.TargetID]event.RulesMap
-	peerClients                []*peerRESTClient // Excludes self
-	allPeerClients             []*peerRESTClient // Includes nil client for self
-}
-
-// GetARNList - returns available ARNs.
-func (sys *NotificationSys) GetARNList(onlyActive bool) []string {
-	arns := []string{}
-	if sys == nil {
-		return arns
-	}
-	region := globalSite.Region
-	for targetID, target := range sys.targetList.TargetMap() {
-		// httpclient target is part of ListenNotification
-		// which doesn't need to be listed as part of the ARN list
-		// This list is only meant for external targets, filter
-		// this out pro-actively.
-		if !strings.HasPrefix(targetID.ID, "httpclient+") {
-			if onlyActive && !target.HasQueueStore() {
-				if _, err := target.IsActive(); err != nil {
-					continue
-				}
-			}
-			arns = append(arns, targetID.ToARN(region).String())
-		}
-	}
-
-	return arns
+	peerClients    []*peerRESTClient // Excludes self
+	allPeerClients []*peerRESTClient // Includes nil client for self
 }
 
 // NotificationPeerErr returns error associated for a remote peer.
@@ -171,7 +139,7 @@ func (sys *NotificationSys) LoadPolicy(policyName string) []NotificationPeerErr
 }
 
 // LoadPolicyMapping - reloads a policy mapping across all peers
-func (sys *NotificationSys) LoadPolicyMapping(userOrGroup string, isGroup bool) []NotificationPeerErr {
+func (sys *NotificationSys) LoadPolicyMapping(userOrGroup string, userType IAMUserType, isGroup bool) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
@@ -179,7 +147,7 @@ func (sys *NotificationSys) LoadPolicyMapping(userOrGroup string, isGroup bool)
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.LoadPolicyMapping(userOrGroup, isGroup)
+			return client.LoadPolicyMapping(userOrGroup, userType, isGroup)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -550,7 +518,7 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName
 	globalReplicationStats.Delete(bucketName)
 	globalBucketMetadataSys.Remove(bucketName)
 	globalBucketTargetSys.Delete(bucketName)
-	globalNotificationSys.RemoveNotification(bucketName)
+	globalEventNotifier.RemoveNotification(bucketName)
 	globalBucketConnStats.delete(bucketName)
 	if localMetacacheMgr != nil {
 		localMetacacheMgr.deleteBucketCache(bucketName)
@@ -684,150 +652,6 @@ func (sys *NotificationSys) LoadTransitionTierConfig(ctx context.Context) {
 	}
 }
 
-// Loads notification policies for all buckets into NotificationSys.
-func (sys *NotificationSys) set(bucket BucketInfo, meta BucketMetadata) {
-	config := meta.notificationConfig
-	if config == nil {
-		return
-	}
-	config.SetRegion(globalSite.Region)
-	if err := config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
-		if _, ok := err.(*event.ErrARNNotFound); !ok {
-			logger.LogIf(GlobalContext, err)
-		}
-	}
-	sys.AddRulesMap(bucket.Name, config.ToRulesMap())
-}
-
-// InitBucketTargets - initializes notification system from notification.xml of all buckets.
-func (sys *NotificationSys) InitBucketTargets(ctx context.Context, objAPI ObjectLayer) error {
-	if objAPI == nil {
-		return errServerNotInitialized
-	}
-
-	// In gateway mode, notifications are not supported - except NAS gateway.
-	if globalIsGateway && !objAPI.IsNotificationSupported() {
-		return nil
-	}
-
-	logger.LogIf(ctx, sys.targetList.Add(globalConfigTargetList.Targets()...))
-
-	go func() {
-		for res := range sys.targetResCh {
-			if res.Err != nil {
-				reqInfo := &logger.ReqInfo{}
-				reqInfo.AppendTags("targetID", res.ID.Name)
-				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
-			}
-		}
-	}()
-
-	return nil
-}
-
-// AddRulesMap - adds rules map for bucket name.
-func (sys *NotificationSys) AddRulesMap(bucketName string, rulesMap event.RulesMap) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	rulesMap = rulesMap.Clone()
-
-	for _, targetRulesMap := range sys.bucketRemoteTargetRulesMap[bucketName] {
-		rulesMap.Add(targetRulesMap)
-	}
-
-	// Do not add for an empty rulesMap.
-	if len(rulesMap) == 0 {
-		delete(sys.bucketRulesMap, bucketName)
-	} else {
-		sys.bucketRulesMap[bucketName] = rulesMap
-	}
-}
-
-// RemoveRulesMap - removes rules map for bucket name.
-func (sys *NotificationSys) RemoveRulesMap(bucketName string, rulesMap event.RulesMap) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	sys.bucketRulesMap[bucketName].Remove(rulesMap)
-	if len(sys.bucketRulesMap[bucketName]) == 0 {
-		delete(sys.bucketRulesMap, bucketName)
-	}
-}
-
-// ConfiguredTargetIDs - returns list of configured target id's
-func (sys *NotificationSys) ConfiguredTargetIDs() []event.TargetID {
-	if sys == nil {
-		return nil
-	}
-
-	sys.RLock()
-	defer sys.RUnlock()
-
-	var targetIDs []event.TargetID
-	for _, rmap := range sys.bucketRulesMap {
-		for _, rules := range rmap {
-			for _, targetSet := range rules {
-				for id := range targetSet {
-					targetIDs = append(targetIDs, id)
-				}
-			}
-		}
-	}
-	// Filter out targets configured via env
-	var tIDs []event.TargetID
-	for _, targetID := range targetIDs {
-		if !globalEnvTargetList.Exists(targetID) {
-			tIDs = append(tIDs, targetID)
-		}
-	}
-	return tIDs
-}
-
-// RemoveNotification - removes all notification configuration for bucket name.
-func (sys *NotificationSys) RemoveNotification(bucketName string) {
-	sys.Lock()
-	defer sys.Unlock()
-
-	delete(sys.bucketRulesMap, bucketName)
-
-	targetIDSet := event.NewTargetIDSet()
-	for targetID := range sys.bucketRemoteTargetRulesMap[bucketName] {
-		targetIDSet[targetID] = struct{}{}
-		delete(sys.bucketRemoteTargetRulesMap[bucketName], targetID)
-	}
-	sys.targetList.Remove(targetIDSet)
-
-	delete(sys.bucketRemoteTargetRulesMap, bucketName)
-}
-
-// RemoveAllRemoteTargets - closes and removes all notification targets.
-func (sys *NotificationSys) RemoveAllRemoteTargets() {
-	sys.Lock()
-	defer sys.Unlock()
-
-	for _, targetMap := range sys.bucketRemoteTargetRulesMap {
-		targetIDSet := event.NewTargetIDSet()
-		for k := range targetMap {
-			targetIDSet[k] = struct{}{}
-		}
-		sys.targetList.Remove(targetIDSet)
-	}
-}
-
-// Send - sends event data to all matching targets.
-func (sys *NotificationSys) Send(args eventArgs) {
-	sys.RLock()
-	targetIDSet := sys.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name)
-	sys.RUnlock()
-
-	if len(targetIDSet) == 0 {
-		return
-	}
-
-	sys.targetList.Send(args.ToEvent(true), targetIDSet, sys.targetResCh)
-}
-
 // GetCPUs - Get all CPU information.
 func (sys *NotificationSys) GetCPUs(ctx context.Context) []madmin.CPUs {
 	reply := make([]madmin.CPUs, len(sys.peerClients))
@@ -904,7 +728,7 @@ func (sys *NotificationSys) GetOSInfo(ctx context.Context) []madmin.OSInfo {
 }
 
 // GetMetrics - Get metrics from all peers.
-func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType, hosts map[string]struct{}) []madmin.RealtimeMetrics {
+func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType, hosts map[string]struct{}, disks map[string]struct{}) []madmin.RealtimeMetrics {
 	reply := make([]madmin.RealtimeMetrics, len(sys.peerClients))
 
 	g := errgroup.WithNErrs(len(sys.peerClients))
@@ -922,7 +746,7 @@ func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType,
 		index := index
 		g.Go(func() error {
 			var err error
-			reply[index], err = sys.peerClients[index].GetMetrics(ctx, t)
+			reply[index], err = sys.peerClients[index].GetMetrics(ctx, t, disks)
 			return err
 		}, index)
 	}
@@ -1190,117 +1014,11 @@ func NewNotificationSys(endpoints EndpointServerPools) *NotificationSys {
 	// targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.Init()
 	remote, all := newPeerRestClients(endpoints)
 	return &NotificationSys{
-		targetList:                 event.NewTargetList(),
-		targetResCh:                make(chan event.TargetIDResult),
-		bucketRulesMap:             make(map[string]event.RulesMap),
-		bucketRemoteTargetRulesMap: make(map[string]map[event.TargetID]event.RulesMap),
-		peerClients:                remote,
-		allPeerClients:             all,
+		peerClients:    remote,
+		allPeerClients: all,
 	}
 }
 
-type eventArgs struct {
-	EventName    event.Name
-	BucketName   string
-	Object       ObjectInfo
-	ReqParams    map[string]string
-	RespElements map[string]string
-	Host         string
-	UserAgent    string
-}
-
-// ToEvent - converts to notification event.
-func (args eventArgs) ToEvent(escape bool) event.Event {
-	eventTime := UTCNow()
-	uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
-
-	respElements := map[string]string{
-		"x-amz-request-id": args.RespElements["requestId"],
-		"x-minio-origin-endpoint": func() string {
-			if globalMinioEndpoint != "" {
-				return globalMinioEndpoint
-			}
-			return getAPIEndpoints()[0]
-		}(), // MinIO specific custom elements.
-	}
-	// Add deployment as part of
-	if globalDeploymentID != "" {
-		respElements["x-minio-deployment-id"] = globalDeploymentID
-	}
-	if args.RespElements["content-length"] != "" {
-		respElements["content-length"] = args.RespElements["content-length"]
-	}
-	keyName := args.Object.Name
-	if escape {
-		keyName = url.QueryEscape(args.Object.Name)
-	}
-	newEvent := event.Event{
-		EventVersion:      "2.0",
-		EventSource:       "minio:s3",
-		AwsRegion:         args.ReqParams["region"],
-		EventTime:         eventTime.Format(event.AMZTimeFormat),
-		EventName:         args.EventName,
-		UserIdentity:      event.Identity{PrincipalID: args.ReqParams["principalId"]},
-		RequestParameters: args.ReqParams,
-		ResponseElements:  respElements,
-		S3: event.Metadata{
-			SchemaVersion:   "1.0",
-			ConfigurationID: "Config",
-			Bucket: event.Bucket{
-				Name:          args.BucketName,
-				OwnerIdentity: event.Identity{PrincipalID: args.ReqParams["principalId"]},
-				ARN:           policy.ResourceARNPrefix + args.BucketName,
-			},
-			Object: event.Object{
-				Key:       keyName,
-				VersionID: args.Object.VersionID,
-				Sequencer: uniqueID,
-			},
-		},
-		Source: event.Source{
-			Host:      args.Host,
-			UserAgent: args.UserAgent,
-		},
-	}
-
-	if args.EventName != event.ObjectRemovedDelete && args.EventName != event.ObjectRemovedDeleteMarkerCreated {
-		newEvent.S3.Object.ETag = args.Object.ETag
-		newEvent.S3.Object.Size = args.Object.Size
-		newEvent.S3.Object.ContentType = args.Object.ContentType
-		newEvent.S3.Object.UserMetadata = make(map[string]string, len(args.Object.UserDefined))
-		for k, v := range args.Object.UserDefined {
-			if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
-				continue
-			}
-			newEvent.S3.Object.UserMetadata[k] = v
-		}
-	}
-
-	return newEvent
-}
-
-func sendEvent(args eventArgs) {
-	args.Object.Size, _ = args.Object.GetActualSize()
-
-	// avoid generating a notification for REPLICA creation event.
-	if _, ok := args.ReqParams[xhttp.MinIOSourceReplicationRequest]; ok {
-		return
-	}
-	// remove sensitive encryption entries in metadata.
-	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
-	crypto.RemoveInternalEntries(args.Object.UserDefined)
-
-	// globalNotificationSys is not initialized in gateway mode.
-	if globalNotificationSys == nil {
-		return
-	}
-	if globalHTTPListen.NumSubscribers(args.EventName) > 0 {
-		globalHTTPListen.Publish(args.ToEvent(false))
-	}
-
-	globalNotificationSys.Send(args)
-}
-
 // GetBandwidthReports - gets the bandwidth report from all nodes including self.
 func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...string) madmin.BucketBandwidthReport {
 	reports := make([]*madmin.BucketBandwidthReport, len(sys.peerClients))

cmd/object-api-errors.go

@@ -422,11 +422,18 @@ func (e BucketRemoteTargetNotFound) Error() string {
 	return "Remote target not found: " + e.Bucket
 }
 
-// BucketRemoteConnectionErr remote target connection failure.
-type BucketRemoteConnectionErr GenericError
+// RemoteTargetConnectionErr remote target connection failure.
+type RemoteTargetConnectionErr struct {
+	Err      error
+	Bucket   string
+	Endpoint string
+}
 
-func (e BucketRemoteConnectionErr) Error() string {
-	return fmt.Sprintf("Remote service endpoint or target bucket not available: %s \n\t%s", e.Bucket, e.Err.Error())
+func (e RemoteTargetConnectionErr) Error() string {
+	if e.Bucket != "" {
+		return fmt.Sprintf("Remote service endpoint offline or target bucket/remote service credentials invalid: %s \n\t%s", e.Bucket, e.Err.Error())
+	}
+	return fmt.Sprintf("Remote service endpoint %s not available\n\t%s", e.Endpoint, e.Err.Error())
 }
 
 // BucketRemoteAlreadyExists remote already exists for this target type.

cmd/object-api-interface.go

@@ -79,7 +79,6 @@ type ObjectOptions struct {
 	// mainly set for certain WRITE operations.
 	SkipDecommissioned bool
 
-	WalkAscending   bool                     // return Walk results in ascending order of versions
 	PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix
 
 	// IndexCB will return any index created but the compression.

cmd/object-api-options.go

@@ -169,7 +169,11 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
 		return opts, err
 	}
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object)
-	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(bucket, object)
+	// Objects matching prefixes should not leave delete markers,
+	// dramatically reduces namespace pollution while keeping the
+	// benefits of replication, make sure to apply version suspension
+	// only at bucket level instead.
+	opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket)
 	delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker))
 	if delMarker != "" {
 		switch delMarker {

cmd/object-handlers.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"encoding/hex"
+	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
@@ -417,12 +418,11 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 
 		return checkPreconditions(ctx, w, r, oi, opts)
 	}
-
+	var proxy proxyResult
 	gr, err := getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	if err != nil {
 		var (
 			reader *GetObjectReader
-			proxy  proxyResult
 			perr   error
 		)
 		proxytgts := getProxyTargets(ctx, bucket, object, opts)
@@ -491,6 +491,10 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		}
 	}
 
+	// Queue failed/pending replication automatically
+	if !proxy.Proxy {
+		QueueReplicationHeal(ctx, bucket, objInfo)
+	}
 	// filter object lock metadata if permission does not permit
 	getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
 	legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
@@ -659,11 +663,9 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 	rangeHeader := r.Header.Get(xhttp.Range)
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+	var proxy proxyResult
 	if err != nil {
-		var (
-			proxy proxyResult
-			oi    ObjectInfo
-		)
+		var oi ObjectInfo
 		// proxy HEAD to replication target if active-active replication configured on bucket
 		proxytgts := getProxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
@@ -690,6 +692,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 					w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)}
 				}
 			}
+			QueueReplicationHeal(ctx, bucket, objInfo)
 			writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 			return
 		}
@@ -714,6 +717,11 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		}
 	}
 
+	// Queue failed/pending replication automatically
+	if !proxy.Proxy {
+		QueueReplicationHeal(ctx, bucket, objInfo)
+	}
+
 	// filter object lock metadata if permission does not permit
 	getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object)
 	legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object)
@@ -3059,6 +3067,16 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
 	})
 }
 
+// ObjectTagSet key value tags
+type ObjectTagSet struct {
+	Tags []tags.Tag `xml:"Tag"`
+}
+
+type objectTagging struct {
+	XMLName xml.Name      `xml:"Tagging"`
+	TagSet  *ObjectTagSet `xml:"TagSet"`
+}
+
 // GetObjectTaggingHandler - GET object tagging
 func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetObjectTagging")
@@ -3096,7 +3114,7 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	// Get object tags
-	tags, err := objAPI.GetObjectTags(ctx, bucket, object, opts)
+	ot, err := objAPI.GetObjectTags(ctx, bucket, object, opts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -3106,7 +3124,20 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 		w.Header()[xhttp.AmzVersionID] = []string{opts.VersionID}
 	}
 
-	writeSuccessResponseXML(w, encodeResponse(tags))
+	otags := &objectTagging{
+		TagSet: &ObjectTagSet{},
+	}
+
+	var list []tags.Tag
+	for k, v := range ot.ToMap() {
+		list = append(list, tags.Tag{
+			Key:   k,
+			Value: v,
+		})
+	}
+	otags.TagSet.Tags = list
+
+	writeSuccessResponseXML(w, encodeResponse(otags))
 }
 
 // PutObjectTaggingHandler - PUT object tagging

cmd/peer-rest-client.go

@@ -195,9 +195,12 @@ func (client *peerRESTClient) GetMemInfo(ctx context.Context) (info madmin.MemIn
 }
 
 // GetMetrics - fetch metrics from a remote node.
-func (client *peerRESTClient) GetMetrics(ctx context.Context, t madmin.MetricType) (info madmin.RealtimeMetrics, err error) {
+func (client *peerRESTClient) GetMetrics(ctx context.Context, t madmin.MetricType, diskMap map[string]struct{}) (info madmin.RealtimeMetrics, err error) {
 	values := make(url.Values)
 	values.Set(peerRESTTypes, strconv.FormatUint(uint64(t), 10))
+	for disk := range diskMap {
+		values.Set(peerRESTDisk, disk)
+	}
 	respBody, err := client.callWithContext(ctx, peerRESTMethodMetrics, values, nil, -1)
 	if err != nil {
 		return
@@ -338,9 +341,10 @@ func (client *peerRESTClient) LoadPolicy(policyName string) (err error) {
 }
 
 // LoadPolicyMapping - reload a specific policy mapping
-func (client *peerRESTClient) LoadPolicyMapping(userOrGroup string, isGroup bool) error {
+func (client *peerRESTClient) LoadPolicyMapping(userOrGroup string, userType IAMUserType, isGroup bool) error {
 	values := make(url.Values)
 	values.Set(peerRESTUserOrGroup, userOrGroup)
+	values.Set(peerRESTUserType, strconv.Itoa(int(userType)))
 	if isGroup {
 		values.Set(peerRESTIsGroup, "")
 	}

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v24" // Change ServerUpdate to DownloadBinary and CommitBinary
+	peerRESTVersion       = "v26" // Add user-type to LoadPolicyMapping
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -82,6 +82,7 @@ const (
 	peerRESTUserTemp     = "user-temp"
 	peerRESTPolicy       = "policy"
 	peerRESTUserOrGroup  = "user-or-group"
+	peerRESTUserType     = "user-type"
 	peerRESTIsGroup      = "is-group"
 	peerRESTSignal       = "signal"
 	peerRESTSubSys       = "sub-sys"
@@ -91,6 +92,7 @@ const (
 	peerRESTDuration     = "duration"
 	peerRESTStorageClass = "storage-class"
 	peerRESTTypes        = "types"
+	peerRESTDisk         = "disk"
 
 	peerRESTListenBucket = "bucket"
 	peerRESTListenPrefix = "prefix"

cmd/peer-rest-server.go

@@ -118,15 +118,20 @@ func (s *peerRESTServer) LoadPolicyMappingHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	vars := mux.Vars(r)
-	userOrGroup := vars[peerRESTUserOrGroup]
+	userOrGroup := r.Form.Get(peerRESTUserOrGroup)
 	if userOrGroup == "" {
 		s.writeErrorResponse(w, errors.New("user-or-group is missing"))
 		return
 	}
 
+	userType, err := strconv.Atoi(r.Form.Get(peerRESTUserType))
+	if err != nil {
+		s.writeErrorResponse(w, fmt.Errorf("user-type `%s` is invalid: %w", r.Form.Get(peerRESTUserType), err))
+		return
+	}
+
 	_, isGroup := r.Form[peerRESTIsGroup]
-	if err := globalIAMSys.LoadPolicyMapping(r.Context(), objAPI, userOrGroup, isGroup); err != nil {
+	if err := globalIAMSys.LoadPolicyMapping(r.Context(), objAPI, userOrGroup, IAMUserType(userType), isGroup); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -426,10 +431,17 @@ func (s *peerRESTServer) GetMetricsHandler(w http.ResponseWriter, r *http.Reques
 		types = madmin.MetricsAll
 	}
 
+	diskMap := make(map[string]struct{})
+	if r.Form != nil {
+		for _, disk := range r.Form[peerRESTDisk] {
+			diskMap[disk] = struct{}{}
+		}
+	}
+
 	ctx, cancel := context.WithCancel(r.Context())
 	defer cancel()
 
-	info := collectLocalMetrics(types, nil)
+	info := collectLocalMetrics(types, nil, diskMap)
 
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
@@ -498,7 +510,7 @@ func (s *peerRESTServer) DeleteBucketMetadataHandler(w http.ResponseWriter, r *h
 	globalReplicationStats.Delete(bucketName)
 	globalBucketMetadataSys.Remove(bucketName)
 	globalBucketTargetSys.Delete(bucketName)
-	globalNotificationSys.RemoveNotification(bucketName)
+	globalEventNotifier.RemoveNotification(bucketName)
 	globalBucketConnStats.delete(bucketName)
 	if localMetacacheMgr != nil {
 		localMetacacheMgr.deleteBucketCache(bucketName)
@@ -591,7 +603,7 @@ func (s *peerRESTServer) LoadBucketMetadataHandler(w http.ResponseWriter, r *htt
 	globalBucketMetadataSys.Set(bucketName, meta)
 
 	if meta.notificationConfig != nil {
-		globalNotificationSys.AddRulesMap(bucketName, meta.notificationConfig.ToRulesMap())
+		globalEventNotifier.AddRulesMap(bucketName, meta.notificationConfig.ToRulesMap())
 	}
 
 	if meta.bucketTargetConfig != nil {
@@ -688,7 +700,7 @@ func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	globalNotificationSys.AddRulesMap(bucketName, rulesMap)
+	globalEventNotifier.AddRulesMap(bucketName, rulesMap)
 }
 
 // Return disk IDs of all the local disks.
@@ -1314,7 +1326,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCommitBinary).HandlerFunc(httpTraceHdrs(server.CommitBinaryHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeletePolicy).HandlerFunc(httpTraceAll(server.DeletePolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicy).HandlerFunc(httpTraceAll(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicyMapping).HandlerFunc(httpTraceAll(server.LoadPolicyMappingHandler)).Queries(restQueries(peerRESTUserOrGroup)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicyMapping).HandlerFunc(httpTraceAll(server.LoadPolicyMappingHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteUser).HandlerFunc(httpTraceAll(server.DeleteUserHandler)).Queries(restQueries(peerRESTUser)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteServiceAccount).HandlerFunc(httpTraceAll(server.DeleteServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadUser).HandlerFunc(httpTraceAll(server.LoadUserHandler)).Queries(restQueries(peerRESTUser, peerRESTUserTemp)...)

cmd/server-main.go

@@ -249,9 +249,12 @@ func initAllSubsystems() {
 	globalBackgroundHealState = newHealState(false)
 	globalHealStateLK.Unlock()
 
-	// Create new notification system and initialize notification peer targets
+	// Initialize notification peer targets
 	globalNotificationSys = NewNotificationSys(globalEndpoints)
 
+	// Create new notification system
+	globalEventNotifier = NewEventNotifier()
+
 	// Create new bucket metadata system.
 	if globalBucketMetadataSys == nil {
 		globalBucketMetadataSys = NewBucketMetadataSys()
@@ -290,7 +293,7 @@ func initAllSubsystems() {
 	}
 
 	// Create new bucket replication subsytem
-	globalBucketTargetSys = NewBucketTargetSys()
+	globalBucketTargetSys = NewBucketTargetSys(GlobalContext)
 
 	// Create new ILM tier configuration subsystem
 	globalTierConfigMgr = NewTierConfigMgr()
@@ -567,21 +570,24 @@ func serverMain(ctx *cli.Context) {
 		logger.LogIf(GlobalContext, err)
 	}
 
-	if globalBrowserEnabled {
-		srv, err := initConsoleServer()
-		if err != nil {
-			logger.FatalIf(err, "Unable to initialize console service")
-		}
-
-		setConsoleSrv(srv)
-
-		go func() {
-			logger.FatalIf(newConsoleServerFn().Serve(), "Unable to initialize console server")
-		}()
-	}
-
 	// Initialize users credentials and policies in background right after config has initialized.
-	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
+	go func() {
+		globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
+
+		// Initialize
+		if globalBrowserEnabled {
+			srv, err := initConsoleServer()
+			if err != nil {
+				logger.FatalIf(err, "Unable to initialize console service")
+			}
+
+			setConsoleSrv(srv)
+
+			go func() {
+				logger.FatalIf(newConsoleServerFn().Serve(), "Unable to initialize console server")
+			}()
+		}
+	}()
 
 	// Background all other operations such as initializing bucket metadata etc.
 	go func() {
@@ -627,7 +633,7 @@ func serverMain(ctx *cli.Context) {
 		globalSiteReplicationSys.Init(GlobalContext, newObject)
 
 		// Initialize bucket notification targets.
-		globalNotificationSys.InitBucketTargets(GlobalContext, newObject)
+		globalEventNotifier.InitBucketTargets(GlobalContext, newObject)
 
 		// initialize the new disk cache objects.
 		if globalCacheConfig.Enabled {

cmd/server-rlimit.go

@@ -21,7 +21,7 @@ import (
 	"runtime"
 	"runtime/debug"
 
-	"github.com/minio/minio/internal/kernel"
+	"github.com/minio/madmin-go/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sys"
 )

cmd/server-startup-msg.go

@@ -163,7 +163,7 @@ func printEventNotifiers() {
 		return
 	}
 
-	arns := globalNotificationSys.GetARNList(true)
+	arns := globalEventNotifier.GetARNList(true)
 	if len(arns) == 0 {
 		return
 	}

cmd/server_test.go

@@ -28,6 +28,7 @@ import (
 	"net/http"
 	"net/url"
 	"reflect"
+	"runtime"
 	"strings"
 	"sync"
 	"testing"
@@ -1636,6 +1637,81 @@ func (s *TestSuiteCommon) TestListObjectsHandler(c *check) {
 	}
 }
 
+// TestListObjectsSpecialCharactersHandler - Setting valid parameters to List Objects
+// and then asserting the response with the expected one.
+func (s *TestSuiteCommon) TestListObjectsSpecialCharactersHandler(c *check) {
+	if runtime.GOOS == globalWindowsOSName {
+		c.Skip("skip special character test for windows")
+	}
+
+	// generate a random bucket name.
+	bucketName := getRandomBucketName()
+	// HTTP request to create the bucket.
+	request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName),
+		0, nil, s.accessKey, s.secretKey, s.signer)
+	c.Assert(err, nil)
+
+	// execute the HTTP request to create bucket.
+	response, err := s.client.Do(request)
+	c.Assert(err, nil)
+	c.Assert(response.StatusCode, http.StatusOK)
+
+	for _, objectName := range []string{"foo bar 1", "foo bar 2", "foo \x01 bar"} {
+		buffer := bytes.NewReader([]byte("Hello World"))
+		request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName),
+			int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer)
+		c.Assert(err, nil)
+
+		response, err = s.client.Do(request)
+		c.Assert(err, nil)
+		c.Assert(response.StatusCode, http.StatusOK)
+	}
+
+	testCases := []struct {
+		getURL          string
+		expectedStrings []string
+	}{
+		{getListObjectsV1URL(s.endPoint, bucketName, "", "1000", ""), []string{"<Key>foo bar 1</Key>", "<Key>foo bar 2</Key>", "<Key>foo &#x1; bar</Key>"}},
+		{getListObjectsV1URL(s.endPoint, bucketName, "", "1000", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>", "<Key>foo+%01+bar</Key>"}},
+		{
+			getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", ""),
+			[]string{
+				"<Key>foo bar 1</Key>",
+				"<Key>foo bar 2</Key>",
+				"<Key>foo &#x1; bar</Key>",
+				fmt.Sprintf("<Owner><ID>%s</ID><DisplayName>minio</DisplayName></Owner>", globalMinioDefaultOwnerID),
+			},
+		},
+		{
+			getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "true", ""),
+			[]string{
+				"<Key>foo bar 1</Key>",
+				"<Key>foo bar 2</Key>",
+				"<Key>foo &#x1; bar</Key>",
+				fmt.Sprintf("<Owner><ID>%s</ID><DisplayName>minio</DisplayName></Owner>", globalMinioDefaultOwnerID),
+			},
+		},
+		{getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url"), []string{"<Key>foo+bar+1</Key>", "<Key>foo+bar+2</Key>", "<Key>foo+%01+bar</Key>"}},
+	}
+
+	for _, testCase := range testCases {
+		// create listObjectsV1 request with valid parameters
+		request, err = newTestSignedRequest(http.MethodGet, testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer)
+		c.Assert(err, nil)
+		// execute the HTTP request.
+		response, err = s.client.Do(request)
+		c.Assert(err, nil)
+		c.Assert(response.StatusCode, http.StatusOK)
+
+		getContent, err := ioutil.ReadAll(response.Body)
+		c.Assert(err, nil)
+
+		for _, expectedStr := range testCase.expectedStrings {
+			c.Assert(strings.Contains(string(getContent), expectedStr), true)
+		}
+	}
+}
+
 // TestListObjectsHandlerErrors - Setting invalid parameters to List Objects
 // and then asserting the error response with the expected one.
 func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) {

cmd/signals.go

@@ -50,8 +50,8 @@ func handleSignals() {
 		// send signal to various go-routines that they need to quit.
 		cancelGlobalContext()
 
-		if globalNotificationSys != nil {
-			globalNotificationSys.RemoveAllRemoteTargets()
+		if globalEventNotifier != nil {
+			globalEventNotifier.RemoveAllRemoteTargets()
 		}
 
 		if httpServer := newHTTPServerFn(); httpServer != nil {

cmd/site-replication.go

@@ -25,6 +25,7 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net/url"
 	"reflect"
 	"sort"
@@ -1103,6 +1104,9 @@ func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, cha
 			_, err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled)
 		} else {
 			_, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members)
+			if err == nil && updReq.Status != madmin.GroupEnabled {
+				_, err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled)
+			}
 		}
 	}
 	if err != nil {
@@ -1198,7 +1202,7 @@ func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mappi
 		}
 	}
 
-	_, err := globalIAMSys.PolicyDBSet(ctx, mapping.UserOrGroup, mapping.Policy, mapping.IsGroup)
+	_, err := globalIAMSys.PolicyDBSet(ctx, mapping.UserOrGroup, mapping.Policy, IAMUserType(mapping.UserType), mapping.IsGroup)
 	if err != nil {
 		return wrapSRErr(err)
 	}
@@ -1748,6 +1752,34 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 		}
 	}
 
+	// Followed by group policy mapping
+	{
+		// Replicate policy mappings on local to all peers.
+		groupPolicyMap := make(map[string]MappedPolicy)
+		globalIAMSys.store.rlock()
+		errG := globalIAMSys.store.loadMappedPolicies(ctx, unknownIAMUserType, true, groupPolicyMap)
+		globalIAMSys.store.runlock()
+		if errG != nil {
+			return errSRBackendIssue(errG)
+		}
+
+		for group, mp := range groupPolicyMap {
+			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
+				Type: madmin.SRIAMItemPolicyMapping,
+				PolicyMapping: &madmin.SRPolicyMapping{
+					UserOrGroup: group,
+					UserType:    -1,
+					IsGroup:     true,
+					Policy:      mp.Policies,
+				},
+				UpdatedAt: mp.UpdatedAt,
+			})
+			if err != nil {
+				return errSRIAMError(err)
+			}
+		}
+	}
+
 	// Service accounts are the static accounts that should be synced with
 	// valid claims.
 	{
@@ -1809,10 +1841,8 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 	{
 		// Replicate policy mappings on local to all peers.
 		userPolicyMap := make(map[string]MappedPolicy)
-		groupPolicyMap := make(map[string]MappedPolicy)
 		globalIAMSys.store.rlock()
 		errU := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap)
-		errG := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap)
 		globalIAMSys.store.runlock()
 		if errU != nil {
 			return errSRBackendIssue(errU)
@@ -1823,6 +1853,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				Type: madmin.SRIAMItemPolicyMapping,
 				PolicyMapping: &madmin.SRPolicyMapping{
 					UserOrGroup: user,
+					UserType:    int(regUser),
 					IsGroup:     false,
 					Policy:      mp.Policies,
 				},
@@ -1832,45 +1863,25 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				return errSRIAMError(err)
 			}
 		}
-
-		if errG != nil {
-			return errSRBackendIssue(errG)
-		}
-
-		for group, mp := range groupPolicyMap {
-			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
-				Type: madmin.SRIAMItemPolicyMapping,
-				PolicyMapping: &madmin.SRPolicyMapping{
-					UserOrGroup: group,
-					IsGroup:     true,
-					Policy:      mp.Policies,
-				},
-				UpdatedAt: mp.UpdatedAt,
-			})
-			if err != nil {
-				return errSRIAMError(err)
-			}
-		}
 	}
 
 	// and finally followed by policy mappings for for STS users.
 	{
 		// Replicate policy mappings on local to all peers.
-		userPolicyMap := make(map[string]MappedPolicy)
-		groupPolicyMap := make(map[string]MappedPolicy)
+		stsPolicyMap := make(map[string]MappedPolicy)
 		globalIAMSys.store.rlock()
-		errU := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap)
-		errG := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap)
+		errU := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, stsPolicyMap)
 		globalIAMSys.store.runlock()
 		if errU != nil {
 			return errSRBackendIssue(errU)
 		}
 
-		for user, mp := range userPolicyMap {
+		for user, mp := range stsPolicyMap {
 			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
 				Type: madmin.SRIAMItemPolicyMapping,
 				PolicyMapping: &madmin.SRPolicyMapping{
 					UserOrGroup: user,
+					UserType:    int(stsUser),
 					IsGroup:     false,
 					Policy:      mp.Policies,
 				},
@@ -1880,25 +1891,6 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 				return errSRIAMError(err)
 			}
 		}
-
-		if errG != nil {
-			return errSRBackendIssue(errG)
-		}
-
-		for group, mp := range groupPolicyMap {
-			err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
-				Type: madmin.SRIAMItemPolicyMapping,
-				PolicyMapping: &madmin.SRPolicyMapping{
-					UserOrGroup: group,
-					IsGroup:     true,
-					Policy:      mp.Policies,
-				},
-				UpdatedAt: mp.UpdatedAt,
-			})
-			if err != nil {
-				return errSRIAMError(err)
-			}
-		}
 	}
 
 	return nil
@@ -2002,7 +1994,7 @@ func (c *SiteReplicationSys) isEnabled() bool {
 	return c.enabled
 }
 
-var errMissingSRConfig = fmt.Errorf("Site not found in site replication configuration")
+var errMissingSRConfig = fmt.Errorf("unable to find site replication configuration")
 
 // RemovePeerCluster - removes one or more clusters from site replication configuration.
 func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI ObjectLayer, rreq madmin.SRRemoveReq) (st madmin.ReplicateRemoveStatus, err error) {
@@ -2054,7 +2046,8 @@ func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI Ob
 				return
 			}
 			if _, err = admClient.SRPeerRemove(ctx, rreq); err != nil {
-				if errors.As(err, &errMissingSRConfig) {
+				if errors.Is(err, errMissingSRConfig) {
+					// ignore if peer is already removed.
 					return
 				}
 				errs[pi.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", pi.Name, err))
@@ -2214,6 +2207,9 @@ func getAdminClient(endpoint, accessKey, secretKey string) (*madmin.AdminClient,
 	if err != nil {
 		return nil, err
 	}
+	if globalBucketTargetSys.isOffline(epURL) {
+		return nil, RemoteTargetConnectionErr{Endpoint: epURL.String(), Err: fmt.Errorf("remote target is offline for endpoint %s", epURL.String())}
+	}
 	client, err := madmin.New(epURL.Host, accessKey, secretKey, epURL.Scheme == "https")
 	if err != nil {
 		return nil, err
@@ -2227,6 +2223,10 @@ func getS3Client(pc madmin.PeerSite) (*minioClient.Client, error) {
 	if err != nil {
 		return nil, err
 	}
+	if globalBucketTargetSys.isOffline(ep) {
+		return nil, RemoteTargetConnectionErr{Endpoint: ep.String(), Err: fmt.Errorf("remote target is offline for endpoint %s", ep.String())}
+	}
+
 	return minioClient.New(ep.Host, &minioClient.Options{
 		Creds:     credentials.NewStaticV4(pc.AccessKey, pc.SecretKey, ""),
 		Secure:    ep.Scheme == "https",
@@ -3502,7 +3502,37 @@ func (c *SiteReplicationSys) PeerEditReq(ctx context.Context, arg madmin.PeerInf
 
 const siteHealTimeInterval = 10 * time.Second
 
+var siteReplicationHealLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{
+	timeout:       30 * time.Second,
+	minimum:       10 * time.Second,
+	retryInterval: time.Second,
+})
+
 func (c *SiteReplicationSys) startHealRoutine(ctx context.Context, objAPI ObjectLayer) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	// Run the site replication healing in a loop
+	for {
+		c.healRoutine(ctx, objAPI)
+		duration := time.Duration(r.Float64() * float64(time.Minute))
+		if duration < time.Second {
+			// Make sure to sleep atleast a second to avoid high CPU ticks.
+			duration = time.Second
+		}
+		time.Sleep(duration)
+	}
+}
+
+func (c *SiteReplicationSys) healRoutine(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only one node running site replication on the cluster.
+	locker := objAPI.NewNSLock(minioMetaBucket, "site-replication/heal.lock")
+	lkctx, err := locker.GetLock(ctx, siteReplicationHealLockTimeout)
+	if err != nil {
+		return
+	}
+	ctx = lkctx.Context()
+	defer lkctx.Cancel()
+	// No unlock for "leader" lock.
+
 	healTimer := time.NewTimer(siteHealTimeInterval)
 	defer healTimer.Stop()
 
@@ -4312,16 +4342,10 @@ func (c *SiteReplicationSys) healIAMSystem(ctx context.Context, objAPI ObjectLay
 		c.healGroups(ctx, objAPI, group, info)
 	}
 	for user := range info.UserStats {
-		c.healUserPolicies(ctx, objAPI, user, info, false)
+		c.healUserPolicies(ctx, objAPI, user, info)
 	}
 	for group := range info.GroupStats {
-		c.healGroupPolicies(ctx, objAPI, group, info, false)
-	}
-	for user := range info.UserStats {
-		c.healUserPolicies(ctx, objAPI, user, info, true)
-	}
-	for group := range info.GroupStats {
-		c.healGroupPolicies(ctx, objAPI, group, info, true)
+		c.healGroupPolicies(ctx, objAPI, group, info)
 	}
 
 	return nil
@@ -4383,7 +4407,7 @@ func (c *SiteReplicationSys) healPolicies(ctx context.Context, objAPI ObjectLaye
 }
 
 // heal user policy mappings present on this site to peers, provided current cluster has the most recent update.
-func (c *SiteReplicationSys) healUserPolicies(ctx context.Context, objAPI ObjectLayer, user string, info srStatusInfo, svcAcct bool) error {
+func (c *SiteReplicationSys) healUserPolicies(ctx context.Context, objAPI ObjectLayer, user string, info srStatusInfo) error {
 	// create user policy mapping on peer cluster if missing
 	us := info.UserStats[user]
 
@@ -4443,7 +4467,7 @@ func (c *SiteReplicationSys) healUserPolicies(ctx context.Context, objAPI Object
 }
 
 // heal group policy mappings present on this site to peers, provided current cluster has the most recent update.
-func (c *SiteReplicationSys) healGroupPolicies(ctx context.Context, objAPI ObjectLayer, group string, info srStatusInfo, svcAcct bool) error {
+func (c *SiteReplicationSys) healGroupPolicies(ctx context.Context, objAPI ObjectLayer, group string, info srStatusInfo) error {
 	// create group policy mapping on peer cluster if missing
 	gs := info.GroupStats[group]
 
@@ -4683,7 +4707,6 @@ func (c *SiteReplicationSys) healGroups(ctx context.Context, objAPI ObjectLayer,
 			continue
 		}
 		peerName := info.Sites[dID].Name
-
 		if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{
 			Type: madmin.SRIAMItemGroupInfo,
 			GroupInfo: &madmin.SRGroupInfo{

cmd/storage-datatypes.go

@@ -40,9 +40,12 @@ type DiskInfo struct {
 	Used       uint64
 	UsedInodes uint64
 	FreeInodes uint64
+	Major      uint32
+	Minor      uint32
 	FSType     string
 	RootDisk   bool
 	Healing    bool
+	Scanning   bool
 	Endpoint   string
 	MountPath  string
 	ID         string

cmd/storage-datatypes_gen.go

@@ -14,8 +14,8 @@ func (z *DiskInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 13 {
-		err = msgp.ArrayError{Wanted: 13, Got: zb0001}
+	if zb0001 != 16 {
+		err = msgp.ArrayError{Wanted: 16, Got: zb0001}
 		return
 	}
 	z.Total, err = dc.ReadUint64()
@@ -43,6 +43,16 @@ func (z *DiskInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "FreeInodes")
 		return
 	}
+	z.Major, err = dc.ReadUint32()
+	if err != nil {
+		err = msgp.WrapError(err, "Major")
+		return
+	}
+	z.Minor, err = dc.ReadUint32()
+	if err != nil {
+		err = msgp.WrapError(err, "Minor")
+		return
+	}
 	z.FSType, err = dc.ReadString()
 	if err != nil {
 		err = msgp.WrapError(err, "FSType")
@@ -58,6 +68,11 @@ func (z *DiskInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "Healing")
 		return
 	}
+	z.Scanning, err = dc.ReadBool()
+	if err != nil {
+		err = msgp.WrapError(err, "Scanning")
+		return
+	}
 	z.Endpoint, err = dc.ReadString()
 	if err != nil {
 		err = msgp.WrapError(err, "Endpoint")
@@ -88,8 +103,8 @@ func (z *DiskInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *DiskInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 13
-	err = en.Append(0x9d)
+	// array header, size 16
+	err = en.Append(0xdc, 0x0, 0x10)
 	if err != nil {
 		return
 	}
@@ -118,6 +133,16 @@ func (z *DiskInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "FreeInodes")
 		return
 	}
+	err = en.WriteUint32(z.Major)
+	if err != nil {
+		err = msgp.WrapError(err, "Major")
+		return
+	}
+	err = en.WriteUint32(z.Minor)
+	if err != nil {
+		err = msgp.WrapError(err, "Minor")
+		return
+	}
 	err = en.WriteString(z.FSType)
 	if err != nil {
 		err = msgp.WrapError(err, "FSType")
@@ -133,6 +158,11 @@ func (z *DiskInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Healing")
 		return
 	}
+	err = en.WriteBool(z.Scanning)
+	if err != nil {
+		err = msgp.WrapError(err, "Scanning")
+		return
+	}
 	err = en.WriteString(z.Endpoint)
 	if err != nil {
 		err = msgp.WrapError(err, "Endpoint")
@@ -164,16 +194,19 @@ func (z *DiskInfo) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *DiskInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 13
-	o = append(o, 0x9d)
+	// array header, size 16
+	o = append(o, 0xdc, 0x0, 0x10)
 	o = msgp.AppendUint64(o, z.Total)
 	o = msgp.AppendUint64(o, z.Free)
 	o = msgp.AppendUint64(o, z.Used)
 	o = msgp.AppendUint64(o, z.UsedInodes)
 	o = msgp.AppendUint64(o, z.FreeInodes)
+	o = msgp.AppendUint32(o, z.Major)
+	o = msgp.AppendUint32(o, z.Minor)
 	o = msgp.AppendString(o, z.FSType)
 	o = msgp.AppendBool(o, z.RootDisk)
 	o = msgp.AppendBool(o, z.Healing)
+	o = msgp.AppendBool(o, z.Scanning)
 	o = msgp.AppendString(o, z.Endpoint)
 	o = msgp.AppendString(o, z.MountPath)
 	o = msgp.AppendString(o, z.ID)
@@ -194,8 +227,8 @@ func (z *DiskInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 13 {
-		err = msgp.ArrayError{Wanted: 13, Got: zb0001}
+	if zb0001 != 16 {
+		err = msgp.ArrayError{Wanted: 16, Got: zb0001}
 		return
 	}
 	z.Total, bts, err = msgp.ReadUint64Bytes(bts)
@@ -223,6 +256,16 @@ func (z *DiskInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "FreeInodes")
 		return
 	}
+	z.Major, bts, err = msgp.ReadUint32Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Major")
+		return
+	}
+	z.Minor, bts, err = msgp.ReadUint32Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Minor")
+		return
+	}
 	z.FSType, bts, err = msgp.ReadStringBytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err, "FSType")
@@ -238,6 +281,11 @@ func (z *DiskInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "Healing")
 		return
 	}
+	z.Scanning, bts, err = msgp.ReadBoolBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "Scanning")
+		return
+	}
 	z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
 	if err != nil {
 		err = msgp.WrapError(err, "Endpoint")
@@ -269,7 +317,7 @@ func (z *DiskInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *DiskInfo) Msgsize() (s int) {
-	s = 1 + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.StringPrefixSize + len(z.FSType) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.Endpoint) + msgp.StringPrefixSize + len(z.MountPath) + msgp.StringPrefixSize + len(z.ID) + z.Metrics.Msgsize() + msgp.StringPrefixSize + len(z.Error)
+	s = 3 + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint32Size + msgp.Uint32Size + msgp.StringPrefixSize + len(z.FSType) + msgp.BoolSize + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.Endpoint) + msgp.StringPrefixSize + len(z.MountPath) + msgp.StringPrefixSize + len(z.ID) + z.Metrics.Msgsize() + msgp.StringPrefixSize + len(z.Error)
 	return
 }
 

cmd/sts-handlers.go

@@ -843,7 +843,8 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 	claims := make(map[string]interface{})
 	defer logger.AuditLog(ctx, w, r, claims)
 
-	if globalAuthNPlugin == nil {
+	authn := newGlobalAuthNPluginFn()
+	if authn == nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCustomToken' is disabled"))
 		return
 	}
@@ -879,7 +880,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *h
 		return
 	}
 
-	res, err := globalAuthNPlugin.Authenticate(roleArn, token)
+	res, err := authn.Authenticate(roleArn, token)
 	if err != nil {
 		writeSTSErrorResponse(ctx, w, true, ErrSTSInvalidParameterValue, err)
 		return

cmd/xl-storage-disk-id-check.go

@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
-	"os"
 	"strconv"
 	"strings"
 	"sync"
@@ -32,6 +31,7 @@ import (
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/env"
 )
 
 //go:generate stringer -type=storageMetric -trimprefix=storageMetric $GOFILE
@@ -575,12 +575,11 @@ const (
 var diskMaxConcurrent = 512
 
 func init() {
-	if s, ok := os.LookupEnv("_MINIO_DISK_MAX_CONCURRENT"); ok && s != "" {
-		var err error
-		diskMaxConcurrent, err = strconv.Atoi(s)
-		if err != nil {
-			logger.Fatal(err, "invalid _MINIO_DISK_MAX_CONCURRENT value")
-		}
+	s := env.Get("_MINIO_DISK_MAX_CONCURRENT", "512")
+	diskMaxConcurrent, _ = strconv.Atoi(s)
+	if diskMaxConcurrent <= 0 {
+		logger.Info("invalid _MINIO_DISK_MAX_CONCURRENT value: %s, defaulting to '512'", s)
+		diskMaxConcurrent = 512
 	}
 }
 

cmd/xl-storage.go

@@ -32,6 +32,7 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
@@ -101,6 +102,9 @@ type xlStorage struct {
 	// Indexes, will be -1 until assigned a set.
 	poolIndex, setIndex, diskIndex int
 
+	// Indicate of NSScanner is in progress in this disk
+	scanning int32
+
 	formatFileInfo  os.FileInfo
 	formatLegacy    bool
 	formatLastCheck time.Time
@@ -423,6 +427,9 @@ func (s *xlStorage) readMetadata(ctx context.Context, itemPath string) ([]byte,
 }
 
 func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error) {
+	atomic.AddInt32(&s.scanning, 1)
+	defer atomic.AddInt32(&s.scanning, -1)
+
 	// Updates must be closed before we return.
 	defer close(updates)
 	var lc *lifecycle.Lifecycle
@@ -576,6 +583,8 @@ func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) {
 			if err != nil {
 				return dcinfo, err
 			}
+			dcinfo.Major = di.Major
+			dcinfo.Minor = di.Minor
 			dcinfo.Total = di.Total
 			dcinfo.Free = di.Free
 			dcinfo.Used = di.Used
@@ -587,6 +596,7 @@ func (s *xlStorage) DiskInfo(context.Context) (info DiskInfo, err error) {
 			// - if we found an unformatted disk (no 'format.json')
 			// - if we found healing tracker 'healing.bin'
 			dcinfo.Healing = errors.Is(err, errUnformattedDisk) || (s.Healing() != nil)
+			dcinfo.Scanning = atomic.LoadInt32(&s.scanning) == 1
 			dcinfo.ID = diskID
 			return dcinfo, err
 		}

docs/bucket/replication/setup_2site_existing_replication.sh

@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -x
+
 trap 'catch $LINENO' ERR
 
 # shellcheck disable=SC2120
@@ -37,8 +39,10 @@ unset MINIO_KMS_KES_KEY_FILE
 unset MINIO_KMS_KES_ENDPOINT
 unset MINIO_KMS_KES_KEY_NAME
 
-wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc \
-    && chmod +x mc
+if [ ! -f ./mc ]; then
+    wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && \
+        chmod +x mc
+fi
 
 minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
       "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
@@ -66,6 +70,9 @@ done
 ./mc mirror /tmp/data sitea/bucket/
 ./mc version enable sitea/bucket
 
+./mc cp /tmp/data/file_1.txt sitea/bucket/marker
+./mc rm sitea/bucket/marker
+
 ./mc mb siteb/bucket/
 ./mc version enable siteb/bucket/
 
@@ -83,11 +90,49 @@ sleep 1
 sleep 10s ## sleep for 10s idea is that we give 100ms per object.
 
 count=$(./mc replicate resync status sitea/bucket --remote-bucket "${remote_arn}" --json | jq .resyncInfo.target[].replicationCount)
-./mc ls --versions sitea/bucket
-./mc ls --versions siteb/bucket
-if [ $count -ne 10 ]; then
-    echo "resync not complete after 100s unexpected failure"
+
+./mc ls -r --versions sitea/bucket > /tmp/sitea.txt
+./mc ls -r --versions siteb/bucket > /tmp/siteb.txt
+
+out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no missing entries after replication: $out"
+    exit 1
+fi
+
+if [ $count -ne 12 ]; then
+    echo "resync not complete after 10s unexpected failure"
     ./mc diff sitea/bucket siteb/bucket
+    exit 1
+fi
+
+./mc cp /tmp/data/file_1.txt sitea/bucket/marker_new
+./mc rm sitea/bucket/marker_new
+
+sleep 12s ## sleep for 12s idea is that we give 100ms per object.
+
+./mc ls -r --versions sitea/bucket > /tmp/sitea.txt
+./mc ls -r --versions siteb/bucket > /tmp/siteb.txt
+
+out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no 'diff' after replication: $out"
+    exit 1
+fi
+
+./mc rm -r --force --versions sitea/bucket/marker
+sleep 14s ## sleep for 14s idea is that we give 100ms per object.
+
+./mc ls -r --versions sitea/bucket > /tmp/sitea.txt
+./mc ls -r --versions siteb/bucket > /tmp/siteb.txt
+
+out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt)
+ret=$?
+if [ $ret -ne 0 ]; then
+    echo "BUG: expected no 'diff' after replication: $out"
+    exit 1
 fi
 
 catch

docs/bucket/replication/setup_3site_replication.sh

@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+if [ -n "$TEST_DEBUG" ]; then
+    set -x
+fi
+
 trap 'catch $LINENO' ERR
 
 # shellcheck disable=SC2120

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-08-05T23-27-09Z
+  image: quay.io/minio/minio:RELEASE.2022-08-22T23-53-06Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/site-replication/run-multi-site-minio-idp.sh

@@ -335,7 +335,7 @@ kill -9 ${site1_pid}
 ./mc rb minio2/bucket2
 # Restart minio1 instance
 minio server --config-dir /tmp/minio-internal --address ":9001" /tmp/minio-internal-idp1/{1...4} >/tmp/minio1_1.log 2>&1 &
-sleep 30
+sleep 40
 
 # Test whether most recent tag update on minio2 is replicated to minio1
 val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key )

docs/tls/README.md

@@ -47,7 +47,7 @@ Download [`certgen`](https://github.com/minio/certgen/releases/latest) for your
 `certgen` is a simple *Go* tool to generate self-signed certificates, and provides SAN certificates with DNS and IP entries:
 
 ```sh
-./certgen -ca -host "10.10.0.3,10.10.0.4,10.10.0.5"
+./certgen -host "10.10.0.3,10.10.0.4,10.10.0.5"
 ```
 
 A response similar to this one should be displayed:

go.mod

@@ -13,7 +13,7 @@ require (
 	github.com/buger/jsonparser v1.1.1
 	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cheggaaa/pb v1.0.29
-	github.com/coredns/coredns v1.9.2
+	github.com/coredns/coredns v1.9.3
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/cosnicolaou/pbzip2 v1.0.1
 	github.com/dchest/siphash v1.2.1
@@ -41,20 +41,21 @@ require (
 	github.com/klauspost/readahead v1.4.0
 	github.com/klauspost/reedsolomon v1.10.0
 	github.com/lib/pq v1.10.4
-	github.com/miekg/dns v1.1.48
-	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.19.4
+	github.com/miekg/dns v1.1.49
+	github.com/minio/cli v1.23.0
+	github.com/minio/console v0.20.0
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.20.0
-	github.com/minio/madmin-go v1.4.13
+	github.com/minio/madmin-go v1.4.25
 	github.com/minio/minio-go/v7 v7.0.34
 	github.com/minio/pkg v1.3.0
 	github.com/minio/selfupdate v0.5.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.4.2
 	github.com/minio/sio v0.3.0
+	github.com/minio/xxml v0.0.3
 	github.com/minio/zipindex v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/nats-io/nats-server/v2 v2.7.4
@@ -72,7 +73,7 @@ require (
 	github.com/rs/cors v1.7.0
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/secure-io/sio-go v0.3.1
-	github.com/shirou/gopsutil/v3 v3.22.6
+	github.com/shirou/gopsutil/v3 v3.22.7
 	github.com/streadway/amqp v1.0.0
 	github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e
 	github.com/valyala/bytebufferpool v1.0.0
@@ -87,7 +88,7 @@ require (
 	golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5
 	golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f
 	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
-	google.golang.org/api v0.78.0
+	google.golang.org/api v0.81.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -136,7 +137,7 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/googleapis/gax-go/v2 v2.3.0 // indirect
+	github.com/googleapis/gax-go/v2 v2.4.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -164,7 +165,7 @@ require (
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/minio/colorjson v1.0.2 // indirect
 	github.com/minio/filepath v1.0.0 // indirect
-	github.com/minio/mc v0.0.0-20220805080128-351d021b924b // indirect
+	github.com/minio/mc v0.0.0-20220818165341-8c239d16aa37 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
 	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -202,20 +203,20 @@ require (
 	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.4 // indirect
-	go.mongodb.org/mongo-driver v1.8.4 // indirect
+	go.mongodb.org/mongo-driver v1.9.1 // indirect
 	go.opencensus.io v0.23.0 // indirect
 	go.uber.org/goleak v1.1.12 // indirect
 	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/mod v0.5.1 // indirect
 	golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.8 // indirect
-	golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f // indirect
+	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e // indirect
-	google.golang.org/grpc v1.46.0 // indirect
+	google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect
+	google.golang.org/grpc v1.46.2 // indirect
 	google.golang.org/protobuf v1.28.0 // indirect
 	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
 	gopkg.in/ini.v1 v1.66.6 // indirect

go.sum

@@ -162,8 +162,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
 github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ=
 github.com/containerd/console v1.0.3 h1:lIr7SlA5PxZyMV30bDW0MGbiOPXwc63yRuCP0ARubLw=
 github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/coredns/coredns v1.9.2 h1:r1uPYQ/HKQq8zoQ3NP2V4k1hxb3Yw2xN9AXcXzofh6U=
-github.com/coredns/coredns v1.9.2/go.mod h1:U44W7RM94WPp8soWjsm8g08oOQ1A6D1xu4VyCYJ79cc=
+github.com/coredns/coredns v1.9.3 h1:eYvVtY9n6qZhHGtQaiOu6ku87Mz9kkSP/6RvFZ48VtA=
+github.com/coredns/coredns v1.9.3/go.mod h1:AS3cwa8gzBsPEqNI6p6LAVNaF7/0SgjjlQQ9JSrzncQ=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-oidc v2.1.0+incompatible h1:sdJrfw8akMnCuUlaZU3tE/uYXFgfqom8DBE9so9EBsM=
@@ -417,8 +417,9 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m
 github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0=
 github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM=
 github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM=
-github.com/googleapis/gax-go/v2 v2.3.0 h1:nRJtk3y8Fm770D42QV6T90ZnvFZyk7agSo3Q+Z9p3WI=
 github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM=
+github.com/googleapis/gax-go/v2 v2.4.0 h1:dS9eYAjhrE2RjmzYw2XAPvcXfmcQLtFEQWn0CR82awk=
+github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gopherjs/gopherjs v0.0.0-20220104163920-15ed2e8cf2bd/go.mod h1:cz9oNYuRUWGdHmLF2IodMLkAhcPtXeULvcBNagUrxTI=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
@@ -601,15 +602,15 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/miekg/dns v1.1.48 h1:Ucfr7IIVyMBz4lRE8qmGUuZ4Wt3/ZGu9hmcMT3Uu4tQ=
-github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
+github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
+github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 github.com/minio/argon2 v1.0.0/go.mod h1:XtOGJ7MjwUJDPtCqqrisx5QwVB/jDx+adQHigJVsQHQ=
-github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
-github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
+github.com/minio/cli v1.23.0 h1:hNuf8xi/JU5EJRX+T38e2zcnh2EpdVqs8aH4lNXWe3w=
+github.com/minio/cli v1.23.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.2 h1:Em3IM68MTm3h+Oxa0nxrV9VQqDgbxvC5iq5A+pqzDeI=
 github.com/minio/colorjson v1.0.2/go.mod h1:JWxcL2n8T8JVf+NY6awl6kn5nK49aAzHOeQEM33dL0k=
-github.com/minio/console v0.19.4 h1:2sSXCU4z95QEaLQOTbSX80BNZOoKj7Jw4DPid+7oPp4=
-github.com/minio/console v0.19.4/go.mod h1:1qeA/u15v0TE/uC27o/jVK6MCpGpK97avKxp+SKBsuI=
+github.com/minio/console v0.20.0 h1:PqS1Iqs3iZH2KtHBWdw9iINlx5UX/eQuWts3zrE/3wY=
+github.com/minio/console v0.20.0/go.mod h1:EQRE16zX6DOTsvH4RUFKQOF3wpNf5I+MJj1NYG/j+KI=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/dperf v0.4.2 h1:u2fcdXViX3+alZcEYSOh1crNbGtVU5drQ24lP3Xx6j8=
@@ -622,10 +623,10 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
 github.com/minio/kes v0.20.0 h1:1tyC51Rr8zTregTESuT/QN/iebNMX7B9t7d3xLNMEpE=
 github.com/minio/kes v0.20.0/go.mod h1:3FW1BQkMGQW78yhy+69tUq5bdcf5rnXJizyeKB9a/tc=
 github.com/minio/madmin-go v1.3.5/go.mod h1:vGKGboQgGIWx4DuDUaXixjlIEZOCIp6ivJkQoiVaACc=
-github.com/minio/madmin-go v1.4.13 h1:ga0vCrafxflxSb8nBnG+FIiu7MMM++6fVVZh+Ej+xzo=
-github.com/minio/madmin-go v1.4.13/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
-github.com/minio/mc v0.0.0-20220805080128-351d021b924b h1:ikMXncKqNE/0acH6us6yy3v+gJBP7nGv/3Rc9F7vRio=
-github.com/minio/mc v0.0.0-20220805080128-351d021b924b/go.mod h1:YUXIqqgGfFknByv0eeJSMBQl/WGuEN0XkpW68/ghBm0=
+github.com/minio/madmin-go v1.4.25 h1:IEJpsTXlHz18/yRI5HxY84KaJdDjUUYdMiC5f8fId4g=
+github.com/minio/madmin-go v1.4.25/go.mod h1:ez87VmMtsxP7DRxjKJKD4RDNW+nhO2QF9KSzwxBDQ98=
+github.com/minio/mc v0.0.0-20220818165341-8c239d16aa37 h1:UE9kHgdza2HB4DDx0nIn0jHRl6YASUomVoKjciMfqjM=
+github.com/minio/mc v0.0.0-20220818165341-8c239d16aa37/go.mod h1:O3pvs5/n57bZ0mpcG/skBuD4pHX6m+wVbW7lw8LO5go=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
 github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
@@ -644,6 +645,8 @@ github.com/minio/simdjson-go v0.4.2 h1:+5rVznvslY/oLIGOO0M7y2g0D8DnFbFZyF2Nr+KT5
 github.com/minio/simdjson-go v0.4.2/go.mod h1:w6ptSSCcUPtqAOOdDVywJkX+8CyYzCdjrTCir0l6BVg=
 github.com/minio/sio v0.3.0 h1:syEFBewzOMOYVzSTFpp1MqpSZk8rUNbz8VIIc+PNzus=
 github.com/minio/sio v0.3.0/go.mod h1:8b0yPp2avGThviy/+OCJBI6OMpvxoUuiLvE6F1lebhw=
+github.com/minio/xxml v0.0.3 h1:ZIpPQpfyG5uZQnqqC0LZuWtPk/WT8G/qkxvO6jb7zMU=
+github.com/minio/xxml v0.0.3/go.mod h1:wcXErosl6IezQIMEWSK/LYC2VS7LJ1dAkgvuyIN3aH4=
 github.com/minio/zipindex v0.2.1 h1:A37vDQJ7Uyp4RHpQEEpintgiIxg0t3npH2CWjLT//u4=
 github.com/minio/zipindex v0.2.1/go.mod h1:s+b/Qyw9JtSEnYfaM4ASOWNO2xGnXCfzQ+SWAzVkVZc=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -805,8 +808,8 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg
 github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc=
 github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs=
 github.com/shirou/gopsutil/v3 v3.21.6/go.mod h1:JfVbDpIBLVzT8oKbvMg9P3wEIMDDpVn+LwHTKj0ST88=
-github.com/shirou/gopsutil/v3 v3.22.6 h1:FnHOFOh+cYAM0C30P+zysPISzlknLC5Z1G4EAElznfQ=
-github.com/shirou/gopsutil/v3 v3.22.6/go.mod h1:EdIubSnZhbAvBS1yJ7Xi+AShB/hxwLHOMz4MCYz7yMs=
+github.com/shirou/gopsutil/v3 v3.22.7 h1:flKnuCMfUUrO+oAvwAd6GKZgnPzr098VA/UJ14nhJd4=
+github.com/shirou/gopsutil/v3 v3.22.7/go.mod h1:s648gW4IywYzUfE/KjXxUsqrqx/T2xO5VqOXxONeRfI=
 github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
 github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -850,7 +853,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
@@ -918,8 +920,8 @@ go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/
 go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
 go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.mongodb.org/mongo-driver v1.8.3/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
-go.mongodb.org/mongo-driver v1.8.4 h1:NruvZPPL0PBcRJKmbswoWSrmHeUvzdxA3GCPfD/NEOA=
-go.mongodb.org/mongo-driver v1.8.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
+go.mongodb.org/mongo-driver v1.9.1 h1:m078y9v7sBItkt1aaoe2YlvWEXcD263e1a4E1fBrJ1c=
+go.mongodb.org/mongo-driver v1.9.1/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
@@ -1053,6 +1055,7 @@ golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
+golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
 golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b h1:PxfKdU9lEEDYjdIzOtC4qFWgkU2rGHdKlKowJSMN9h0=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -1088,8 +1091,9 @@ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
+golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1278,8 +1282,9 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f h1:GGU+dLjvlC3qDwqYgL6UgRmHXhOOgns0bZu2Ty5mm6U=
 golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618=
+golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
 google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
 google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
 google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1317,8 +1322,9 @@ google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/S
 google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
 google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
 google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
-google.golang.org/api v0.78.0 h1:5ewPyCwP43C3i8B6C2Kb+eVAevbnke2xR8VbcSWjS4I=
 google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
+google.golang.org/api v0.81.0 h1:o8WF5AvfidafWbFjsRyupxyEQJNUWxLZJCK5NXrxZZ8=
+google.golang.org/api v0.81.0/go.mod h1:FA6Mb/bZxj706H2j+j2d6mHEEaHBmbbWnkfvmorOCko=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1398,8 +1404,10 @@ google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX
 google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
 google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
-google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e h1:gMjH4zLGs9m+dGzR7qHCHaXMOwsJHJKKkHtyXhtOrJk=
 google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
+google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
+google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd h1:e0TwkXOdbnH/1x5rc5MZ/VYyiZ4v+RdVfrGMqEwT68I=
+google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1428,8 +1436,9 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K
 google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
 google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ=
-google.golang.org/grpc v1.46.0 h1:oCjezcn6g6A75TGoKYBPgKmVBLexhYLM6MebdrPApP8=
 google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
+google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ=
+google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk=
 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 4.0.11
-appVersion: RELEASE.2022-08-05T23-27-09Z
+version: 4.0.13
+appVersion: RELEASE.2022-08-22T23-53-06Z
 keywords:
   - minio
   - storage

helm/minio/README.md

@@ -188,7 +188,8 @@ Description of the configuration parameters used above -
 - `buckets[].policy` - can be one of none|download|upload|public
 - `buckets[].purge` - purge if bucket exists already
 
-33# Create policies after install
+### Create policies after install
+
 Install the chart, specifying the policies you want to create after install:
 
 ```bash

helm/minio/templates/_helper_create_bucket.txt

@@ -50,6 +50,7 @@ createBucket() {
   POLICY=$2
   PURGE=$3
   VERSIONING=$4
+  OBJECTLOCKING=$5
 
   # Purge the bucket, if set & exists
   # Since PURGE is user input, check explicitly for `true`
@@ -64,16 +65,27 @@ createBucket() {
     fi
   fi
 
-  # Create the bucket if it does not exist
-  if ! checkBucketExists $BUCKET ; then
-    echo "Creating bucket '$BUCKET'"
-    ${MC} mb myminio/$BUCKET
+# Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created)
+if ! checkBucketExists $BUCKET ; then
+    if [ ! -z $OBJECTLOCKING ] ; then
+      if [ $OBJECTLOCKING = true ] ; then
+          echo "Creating bucket with OBJECTLOCKING '$BUCKET'"
+          ${MC} mb --with-lock myminio/$BUCKET
+      elif [ $OBJECTLOCKING = false ] ; then
+            echo "Creating bucket '$BUCKET'"
+            ${MC} mb myminio/$BUCKET
+      fi
+  elif [ -z $OBJECTLOCKING ] ; then
+        echo "Creating bucket '$BUCKET'"
+        ${MC} mb myminio/$BUCKET
   else
-    echo "Bucket '$BUCKET' already exists."
+    echo "Bucket '$BUCKET' already exists."  
+  fi
   fi
 
 
-  # set versioning for bucket
+  # set versioning for bucket if objectlocking is disabled or not set
+  if [ -z $OBJECTLOCKING ] ; then
   if [ ! -z $VERSIONING ] ; then
     if [ $VERSIONING = true ] ; then
         echo "Enabling versioning for '$BUCKET'"
@@ -82,10 +94,12 @@ createBucket() {
         echo "Suspending versioning for '$BUCKET'"
         ${MC} version suspend myminio/$BUCKET
     fi
+    fi
   else
       echo "Bucket '$BUCKET' versioning unchanged."
   fi
 
+
   # At this point, the bucket should exist, skip checking for existence
   # Set policy on the bucket
   echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
@@ -104,6 +118,6 @@ connectToMinio $scheme
 {{ $global := . }}
 # Create the buckets
 {{- range .Values.buckets }}
-createBucket {{ tpl .name $global }} {{ .policy }} {{ .purge }} {{ .versioning }}
-{{- end }}
+createBucket {{ tpl .name $global }} {{ .policy }} {{ .purge }} {{ .versioning }} {{ .objectlocking }}
 {{- end }}
+{{- end }}

helm/minio/templates/deployment.yaml

@@ -82,18 +82,19 @@ spec:
             - name: minio-user
               mountPath: "/tmp/credentials"
               readOnly: true
-            {{- if .Values.persistence.enabled }}
             - name: export
               mountPath: {{ .Values.mountPath }}
-              {{- if .Values.persistence.subPath }}
+              {{- if and .Values.persistence.enabled .Values.persistence.subPath }}
               subPath: "{{ .Values.persistence.subPath }}"
               {{- end }}
-            {{- end }}
             {{- if .Values.extraSecret }}
             - name: extra-secret
               mountPath: "/tmp/minio-config-env"
             {{- end }}
             {{- include "minio.tlsKeysVolumeMount" . | indent 12 }}
+            {{- if .Values.extraVolumeMounts }}
+              {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
+            {{- end }}
           ports:
             - name: {{ $scheme }}
               containerPort: {{ .Values.minioAPIPort }}
@@ -123,7 +124,7 @@ spec:
               value: {{ .Values.oidc.configUrl }}
             - name: MINIO_IDENTITY_OPENID_CLIENT_ID
               value: {{ .Values.oidc.clientId }}
-            - name: MINIO_IDENTITY_OPENID_CLIENTs_SECRET
+            - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET
               value: {{ .Values.oidc.clientSecret }}
             - name: MINIO_IDENTITY_OPENID_CLAIM_NAME
               value: {{ .Values.oidc.claimName }}
@@ -192,4 +193,7 @@ spec:
           secret:
             secretName: {{ template "minio.secretName" . }}
         {{- include "minio.tlsKeysVolume" . | indent 8 }}
+        {{- if .Values.extraVolumes }}
+          {{ toYaml .Values.extraVolumes | nindent 8 }}
+        {{- end }}
 {{- end }}

helm/minio/templates/statefulset.yaml

@@ -130,6 +130,9 @@ spec:
               mountPath: "/tmp/minio-config-env"
             {{- end }}
             {{- include "minio.tlsKeysVolumeMount" . | indent 12 }}
+            {{- if .Values.extraVolumeMounts }}
+              {{- toYaml .Values.extraVolumeMounts | nindent 12 }}
+            {{- end }}
           ports:
             - name: {{ $scheme }}
               containerPort: {{ .Values.minioAPIPort }}
@@ -207,6 +210,9 @@ spec:
             secretName: {{ .Values.extraSecret }}
         {{- end }}
         {{- include "minio.tlsKeysVolume" . | indent 8 }}
+        {{- if .Values.extraVolumes }}
+          {{ toYaml .Values.extraVolumes | nindent 8 }}
+        {{- end }}
 {{- if .Values.persistence.enabled }}
   volumeClaimTemplates:
   {{- if gt $drivesPerNode 1 }}

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2022-08-05T23-27-09Z
+  tag: RELEASE.2022-08-22T23-53-06Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,7 +25,7 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2022-08-05T08-01-28Z
+  tag: RELEASE.2022-08-23T05-45-20Z
   pullPolicy: IfNotPresent
 
 ## minio mode, i.e. standalone or distributed or gateway.
@@ -46,6 +46,12 @@ ignoreChartChecksums: false
 ## Additional arguments to pass to minio binary
 extraArgs: []
 
+## Additional volumes to minio container
+extraVolumes: []
+
+## Additional volumeMounts to minio container
+extraVolumeMounts: []
+
 ## Internal port number for MinIO S3 API container
 ## Change service.port to change external port number
 minioAPIPort: "9000"
@@ -297,8 +303,8 @@ policies: []
 #         - "s3:ListBucketMultipartUploads"
 ## Additional Annotations for the Kubernetes Job makePolicyJob
 makePolicyJob:
-  podAnnotations:
-  annotations:
+  podAnnotations: {}
+  annotations: {}
   securityContext:
     enabled: false
     runAsUser: 1000
@@ -333,8 +339,8 @@ users:
 
 ## Additional Annotations for the Kubernetes Job makeUserJob
 makeUserJob:
-  podAnnotations:
-  annotations:
+  podAnnotations: {}
+  annotations: {}
   securityContext:
     enabled: false
     runAsUser: 1000
@@ -362,15 +368,21 @@ buckets:
   #   # set versioning for
   #   # bucket [true|false]
   #   versioning: false
+  #   # set objectlocking for
+  #   # bucket [true|false] NOTE: versioning is enabled by default if you use locking 
+  #   objectlocking: false
   # - name: bucket2
   #   policy: none
   #   purge: false
   #   versioning: true
+  #   # set objectlocking for
+  #   # bucket [true|false] NOTE: versioning is enabled by default if you use locking 
+  #   objectlocking: false
 
 ## Additional Annotations for the Kubernetes Job makeBucketJob
 makeBucketJob:
-  podAnnotations:
-  annotations:
+  podAnnotations: {}
+  annotations: {}
   securityContext:
     enabled: false
     runAsUser: 1000
@@ -392,8 +404,8 @@ customCommands:
 
 ## Additional Annotations for the Kubernetes Job customCommandJob
 customCommandJob:
-  podAnnotations:
-  annotations:
+  podAnnotations: {}
+  annotations: {}
   securityContext:
     enabled: false
     runAsUser: 1000

index.yaml

@@ -1,9 +1,53 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2022-08-22T23-53-06Z
+    created: "2022-08-23T11:17:59.629897343-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 5b86937ca88d9f6046141fdc2b1cc54760435ed92d289cd0a115fa7148781d4e
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-4.0.13.tgz
+    version: 4.0.13
+  - apiVersion: v1
+    appVersion: RELEASE.2022-08-13T21-54-44Z
+    created: "2022-08-23T11:17:59.628879914-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 2d9c227c0f46ea8bdef4d760c212156fd4c6623ddc5406779c569fe925527787
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-4.0.12.tgz
+    version: 4.0.12
   - apiVersion: v1
     appVersion: RELEASE.2022-08-05T23-27-09Z
-    created: "2022-08-06T22:41:27.785801159-07:00"
+    created: "2022-08-23T11:17:59.627610129-07:00"
     description: Multi-Cloud Object Storage
     digest: 6caaffcb636e040cd7e8bc4883a1674a673757f4781c32d53b5ec0f41fea3944
     home: https://min.io
@@ -25,7 +69,7 @@ entries:
     version: 4.0.11
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2022-08-06T22:41:27.784681914-07:00"
+    created: "2022-08-23T11:17:59.626371541-07:00"
     description: Multi-Cloud Object Storage
     digest: 841d87788fb094d6a7d8a91e91821fe1e847bc952e054c781fc93742d112e18a
     home: https://min.io
@@ -47,7 +91,7 @@ entries:
     version: 4.0.10
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2022-08-06T22:41:27.797186807-07:00"
+    created: "2022-08-23T11:17:59.640048178-07:00"
     description: Multi-Cloud Object Storage
     digest: 6f1a78382df3215deac07495a5e7de7009a1153b4cf6cb565630652a69aec4cf
     home: https://min.io
@@ -69,7 +113,7 @@ entries:
     version: 4.0.9
   - apiVersion: v1
     appVersion: RELEASE.2022-07-29T19-40-48Z
-    created: "2022-08-06T22:41:27.795428949-07:00"
+    created: "2022-08-23T11:17:59.638955929-07:00"
     description: Multi-Cloud Object Storage
     digest: d11db37963636922cb778b6bc0ad2ca4724cb391ea7b785995ada52467d7dd83
     home: https://min.io
@@ -91,7 +135,7 @@ entries:
     version: 4.0.8
   - apiVersion: v1
     appVersion: RELEASE.2022-07-26T00-53-03Z
-    created: "2022-08-06T22:41:27.794132186-07:00"
+    created: "2022-08-23T11:17:59.637806034-07:00"
     description: Multi-Cloud Object Storage
     digest: ca775e08c84331bb5029d4d29867d30c16e2c62e897788eb432212a756e91e4e
     home: https://min.io
@@ -113,7 +157,7 @@ entries:
     version: 4.0.7
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2022-08-06T22:41:27.792926994-07:00"
+    created: "2022-08-23T11:17:59.63605872-07:00"
     description: Multi-Cloud Object Storage
     digest: 06542b8f3d149d5908b15de9a8d6f8cf304af0213830be56dc315785d14f9ccd
     home: https://min.io
@@ -135,7 +179,7 @@ entries:
     version: 4.0.6
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2022-08-06T22:41:27.791618898-07:00"
+    created: "2022-08-23T11:17:59.634973716-07:00"
     description: Multi-Cloud Object Storage
     digest: dd2676362f067454a496cdd293609d0c904b08f521625af49f95402a024ba1f5
     home: https://min.io
@@ -157,7 +201,7 @@ entries:
     version: 4.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2022-08-06T22:41:27.790266011-07:00"
+    created: "2022-08-23T11:17:59.63389644-07:00"
     description: Multi-Cloud Object Storage
     digest: bab9ef192d4eda4c572ad0ce0cf551736c847f582d1837d6833ee10543c23167
     home: https://min.io
@@ -179,7 +223,7 @@ entries:
     version: 4.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2022-08-06T22:41:27.788308162-07:00"
+    created: "2022-08-23T11:17:59.632850224-07:00"
     description: Multi-Cloud Object Storage
     digest: c770bb9841c76576e4e8573f78b0ec33e0d729504c9667e67ad62d48df5ed64c
     home: https://min.io
@@ -201,7 +245,7 @@ entries:
     version: 4.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2022-08-06T22:41:27.787031639-07:00"
+    created: "2022-08-23T11:17:59.631653156-07:00"
     description: Multi-Cloud Object Storage
     digest: 95835f4199d963e2a23a2493610b348e6f2ff8b71c1a648c4a3b84af9b7a83eb
     home: https://min.io
@@ -223,7 +267,7 @@ entries:
     version: 4.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-04-30T22-23-53Z
-    created: "2022-08-06T22:41:27.783259224-07:00"
+    created: "2022-08-23T11:17:59.62516544-07:00"
     description: Multi-Cloud Object Storage
     digest: 55a088c403b056e1f055a97426aa11759c3d6cbad38face170fe6cbbec7d568f
     home: https://min.io
@@ -245,7 +289,7 @@ entries:
     version: 4.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-04-26T01-20-24Z
-    created: "2022-08-06T22:41:27.781604559-07:00"
+    created: "2022-08-23T11:17:59.623568334-07:00"
     description: Multi-Cloud Object Storage
     digest: f541237e24336ec3f7f45ae0d523fef694e3a2f9ef648c5b11c15734db6ba2b2
     home: https://min.io
@@ -267,7 +311,7 @@ entries:
     version: 4.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-04-16T04-26-02Z
-    created: "2022-08-06T22:41:27.780277374-07:00"
+    created: "2022-08-23T11:17:59.622515332-07:00"
     description: Multi-Cloud Object Storage
     digest: edc0c3dd6d5246a06b74ba16bb4aff80a6d7225dc9aecf064fd89a8af371b9c1
     home: https://min.io
@@ -289,7 +333,7 @@ entries:
     version: 3.6.6
   - apiVersion: v1
     appVersion: RELEASE.2022-04-12T06-55-35Z
-    created: "2022-08-06T22:41:27.779043161-07:00"
+    created: "2022-08-23T11:17:59.621478578-07:00"
     description: Multi-Cloud Object Storage
     digest: 211e89f6b9eb0b9a3583abaa127be60e1f9717a098e6b2858cb9dc1cc50c1650
     home: https://min.io
@@ -311,7 +355,7 @@ entries:
     version: 3.6.5
   - apiVersion: v1
     appVersion: RELEASE.2022-04-09T15-09-52Z
-    created: "2022-08-06T22:41:27.777988431-07:00"
+    created: "2022-08-23T11:17:59.62030524-07:00"
     description: Multi-Cloud Object Storage
     digest: 534a879d73b370a18b554b93d0930e1c115419619c4ce4ec7dbaae632acacf06
     home: https://min.io
@@ -333,7 +377,7 @@ entries:
     version: 3.6.4
   - apiVersion: v1
     appVersion: RELEASE.2022-03-24T00-43-44Z
-    created: "2022-08-06T22:41:27.776634155-07:00"
+    created: "2022-08-23T11:17:59.619208751-07:00"
     description: Multi-Cloud Object Storage
     digest: 99508b20eb0083a567dcccaf9a6c237e09575ed1d70cd2e8333f89c472d13d75
     home: https://min.io
@@ -355,7 +399,7 @@ entries:
     version: 3.6.3
   - apiVersion: v1
     appVersion: RELEASE.2022-03-17T06-34-49Z
-    created: "2022-08-06T22:41:27.775420038-07:00"
+    created: "2022-08-23T11:17:59.618023919-07:00"
     description: Multi-Cloud Object Storage
     digest: b4cd25611ca322b1d23d23112fdfa6b068fd91eefe0b0663b88ff87ea4282495
     home: https://min.io
@@ -377,7 +421,7 @@ entries:
     version: 3.6.2
   - apiVersion: v1
     appVersion: RELEASE.2022-03-14T18-25-24Z
-    created: "2022-08-06T22:41:27.773769392-07:00"
+    created: "2022-08-23T11:17:59.616310852-07:00"
     description: Multi-Cloud Object Storage
     digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c
     home: https://min.io
@@ -399,7 +443,7 @@ entries:
     version: 3.6.1
   - apiVersion: v1
     appVersion: RELEASE.2022-03-11T23-57-45Z
-    created: "2022-08-06T22:41:27.772455831-07:00"
+    created: "2022-08-23T11:17:59.615289812-07:00"
     description: Multi-Cloud Object Storage
     digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d
     home: https://min.io
@@ -421,7 +465,7 @@ entries:
     version: 3.6.0
   - apiVersion: v1
     appVersion: RELEASE.2022-03-03T21-21-16Z
-    created: "2022-08-06T22:41:27.771245669-07:00"
+    created: "2022-08-23T11:17:59.614241328-07:00"
     description: Multi-Cloud Object Storage
     digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
     home: https://min.io
@@ -443,7 +487,7 @@ entries:
     version: 3.5.9
   - apiVersion: v1
     appVersion: RELEASE.2022-02-26T02-54-46Z
-    created: "2022-08-06T22:41:27.769977384-07:00"
+    created: "2022-08-23T11:17:59.613161505-07:00"
     description: Multi-Cloud Object Storage
     digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
     home: https://min.io
@@ -465,7 +509,7 @@ entries:
     version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-08-06T22:41:27.768725742-07:00"
+    created: "2022-08-23T11:17:59.611988834-07:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -487,7 +531,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-08-06T22:41:27.766562506-07:00"
+    created: "2022-08-23T11:17:59.610662386-07:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -509,7 +553,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2022-08-06T22:41:27.765380334-07:00"
+    created: "2022-08-23T11:17:59.609200041-07:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -531,7 +575,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-08-06T22:41:27.764276392-07:00"
+    created: "2022-08-23T11:17:59.608153417-07:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -553,7 +597,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-08-06T22:41:27.763151587-07:00"
+    created: "2022-08-23T11:17:59.607042521-07:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -575,7 +619,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2022-08-06T22:41:27.762103403-07:00"
+    created: "2022-08-23T11:17:59.606145485-07:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -597,7 +641,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-08-06T22:41:27.760922693-07:00"
+    created: "2022-08-23T11:17:59.605161074-07:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -619,7 +663,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-08-06T22:41:27.759329213-07:00"
+    created: "2022-08-23T11:17:59.60410066-07:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -641,7 +685,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2022-08-06T22:41:27.758454022-07:00"
+    created: "2022-08-23T11:17:59.602675363-07:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -663,7 +707,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2022-08-06T22:41:27.757532746-07:00"
+    created: "2022-08-23T11:17:59.601783461-07:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -685,7 +729,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2022-08-06T22:41:27.756372116-07:00"
+    created: "2022-08-23T11:17:59.600870662-07:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -707,7 +751,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2022-08-06T22:41:27.755228661-07:00"
+    created: "2022-08-23T11:17:59.599858205-07:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -729,7 +773,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2022-08-06T22:41:27.75409215-07:00"
+    created: "2022-08-23T11:17:59.598868829-07:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -751,7 +795,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-08-06T22:41:27.753010077-07:00"
+    created: "2022-08-23T11:17:59.59794271-07:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -773,7 +817,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-08-06T22:41:27.751404631-07:00"
+    created: "2022-08-23T11:17:59.596260123-07:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -795,7 +839,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-08-06T22:41:27.750392657-07:00"
+    created: "2022-08-23T11:17:59.595230706-07:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -817,7 +861,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2022-08-06T22:41:27.749380775-07:00"
+    created: "2022-08-23T11:17:59.59413522-07:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -839,7 +883,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-08-06T22:41:27.748370924-07:00"
+    created: "2022-08-23T11:17:59.593208767-07:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -861,7 +905,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-08-06T22:41:27.747265405-07:00"
+    created: "2022-08-23T11:17:59.592202916-07:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -883,7 +927,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-08-06T22:41:27.746241467-07:00"
+    created: "2022-08-23T11:17:59.591152731-07:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -905,7 +949,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-08-06T22:41:27.744659943-07:00"
+    created: "2022-08-23T11:17:59.58960993-07:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -927,7 +971,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-08-06T22:41:27.743469302-07:00"
+    created: "2022-08-23T11:17:59.588715068-07:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -949,7 +993,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2022-08-06T22:41:27.742460628-07:00"
+    created: "2022-08-23T11:17:59.587765832-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -971,7 +1015,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2022-08-06T22:41:27.741533311-07:00"
+    created: "2022-08-23T11:17:59.586917787-07:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -993,7 +1037,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2022-08-06T22:41:27.740633359-07:00"
+    created: "2022-08-23T11:17:59.585917039-07:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -1015,7 +1059,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2022-08-06T22:41:27.739484596-07:00"
+    created: "2022-08-23T11:17:59.584915986-07:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -1037,7 +1081,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-08-06T22:41:27.737938992-07:00"
+    created: "2022-08-23T11:17:59.58347999-07:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -1059,7 +1103,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-08-06T22:41:27.737042805-07:00"
+    created: "2022-08-23T11:17:59.582521862-07:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -1081,7 +1125,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-08-06T22:41:27.736009219-07:00"
+    created: "2022-08-23T11:17:59.581572145-07:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -1103,7 +1147,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-08-06T22:41:27.734987403-07:00"
+    created: "2022-08-23T11:17:59.580589742-07:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -1125,7 +1169,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-08-06T22:41:27.734104571-07:00"
+    created: "2022-08-23T11:17:59.579720834-07:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -1147,7 +1191,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-08-06T22:41:27.733098277-07:00"
+    created: "2022-08-23T11:17:59.57867667-07:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -1169,7 +1213,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-08-06T22:41:27.731973441-07:00"
+    created: "2022-08-23T11:17:59.577658313-07:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -1191,7 +1235,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2022-08-06T22:41:27.730308483-07:00"
+    created: "2022-08-23T11:17:59.574200392-07:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -1213,7 +1257,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-08-06T22:41:27.729384519-07:00"
+    created: "2022-08-23T11:17:59.573235559-07:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -1235,7 +1279,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-08-06T22:41:27.728447324-07:00"
+    created: "2022-08-23T11:17:59.572389668-07:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -1257,7 +1301,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-08-06T22:41:27.72752056-07:00"
+    created: "2022-08-23T11:17:59.571454277-07:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -1277,7 +1321,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-08-06T22:41:27.726522128-07:00"
+    created: "2022-08-23T11:17:59.570630455-07:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -1297,7 +1341,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-08-06T22:41:27.725622611-07:00"
+    created: "2022-08-23T11:17:59.569846896-07:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -1317,7 +1361,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-08-06T22:41:27.72460789-07:00"
+    created: "2022-08-23T11:17:59.568912724-07:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -1337,7 +1381,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-08-06T22:41:27.723182324-07:00"
+    created: "2022-08-23T11:17:59.567491164-07:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -1357,7 +1401,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-08-06T22:41:27.722107087-07:00"
+    created: "2022-08-23T11:17:59.56653284-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -1377,7 +1421,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-08-06T22:41:27.720990432-07:00"
+    created: "2022-08-23T11:17:59.565486113-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -1397,7 +1441,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2022-08-06T22:41:27.720031849-07:00"
+    created: "2022-08-23T11:17:59.564493352-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -1415,4 +1459,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2022-08-06T22:41:27.718811472-07:00"
+generated: "2022-08-23T11:17:59.563218908-07:00"

internal/bucket/encryption/bucket-sse-config.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"io"
 	"net/http"
+	"strings"
 
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
@@ -102,9 +103,14 @@ func ParseBucketSSEConfig(r io.Reader) (*BucketSSEConfig, error) {
 				return nil, errors.New("MasterKeyID is allowed with aws:kms only")
 			}
 		case AWSKms:
-			if rule.DefaultEncryptionAction.MasterKeyID == "" {
+			keyID := rule.DefaultEncryptionAction.MasterKeyID
+			if keyID == "" {
 				return nil, errors.New("MasterKeyID is missing with aws:kms")
 			}
+			spaces := strings.HasPrefix(keyID, " ") || strings.HasSuffix(keyID, " ")
+			if spaces {
+				return nil, errors.New("MasterKeyID contains unsupported characters")
+			}
 		}
 	}
 
@@ -164,7 +170,7 @@ func (b *BucketSSEConfig) Algo() Algorithm {
 // empty key ID.
 func (b *BucketSSEConfig) KeyID() string {
 	for _, rule := range b.Rules {
-		return rule.DefaultEncryptionAction.MasterKeyID
+		return strings.TrimPrefix(rule.DefaultEncryptionAction.MasterKeyID, crypto.ARNPrefix)
 	}
 	return ""
 }

internal/bucket/encryption/bucket-sse-config_test.go

@@ -62,7 +62,7 @@ func TestParseBucketSSEConfig(t *testing.T) {
 			{
 				DefaultEncryptionAction: EncryptionAction{
 					Algorithm:   AWSKms,
-					MasterKeyID: "arn:aws:kms:us-east-1:1234/5678example",
+					MasterKeyID: "arn:aws:kms:my-minio-key",
 				},
 			},
 		},
@@ -70,6 +70,7 @@ func TestParseBucketSSEConfig(t *testing.T) {
 
 	testCases := []struct {
 		inputXML       string
+		keyID          string
 		expectedErr    error
 		shouldPass     bool
 		expectedConfig *BucketSSEConfig
@@ -83,10 +84,11 @@ func TestParseBucketSSEConfig(t *testing.T) {
 		},
 		// 2. Valid XML SSE-KMS
 		{
-			inputXML:       `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID>arn:aws:kms:us-east-1:1234/5678example</KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`,
+			inputXML:       `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID>arn:aws:kms:my-minio-key</KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`,
 			expectedErr:    nil,
 			shouldPass:     true,
 			expectedConfig: actualKMSConfig,
+			keyID:          "my-minio-key",
 		},
 		// 3. Invalid - more than one rule
 		{
@@ -119,23 +121,33 @@ func TestParseBucketSSEConfig(t *testing.T) {
 			shouldPass:     true,
 			expectedConfig: actualAES256NoNSConfig,
 		},
+		// 8. Space characters in MasterKeyID
+		{
+			inputXML:    `<ServerSideEncryptionConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>aws:kms</SSEAlgorithm><KMSMasterKeyID> arn:aws:kms:my-minio-key </KMSMasterKeyID></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>`,
+			expectedErr: errors.New("MasterKeyID contains unsupported characters"),
+			shouldPass:  false,
+		},
 	}
 
 	for i, tc := range testCases {
-		_, err := ParseBucketSSEConfig(bytes.NewReader([]byte(tc.inputXML)))
+		ssec, err := ParseBucketSSEConfig(bytes.NewReader([]byte(tc.inputXML)))
 		if tc.shouldPass && err != nil {
-			t.Fatalf("Test case %d: Expected to succeed but got %s", i+1, err)
+			t.Errorf("Test case %d: Expected to succeed but got %s", i+1, err)
 		}
 
 		if !tc.shouldPass {
 			if err == nil || err != nil && err.Error() != tc.expectedErr.Error() {
-				t.Fatalf("Test case %d: Expected %s but got %s", i+1, tc.expectedErr, err)
+				t.Errorf("Test case %d: Expected %s but got %s", i+1, tc.expectedErr, err)
 			}
 			continue
 		}
 
+		if tc.keyID != "" && tc.keyID != ssec.KeyID() {
+			t.Errorf("Test case %d: Expected bucket encryption KeyID %s but got %s", i+1, tc.keyID, ssec.KeyID())
+		}
+
 		if expectedXML, err := xml.Marshal(tc.expectedConfig); err != nil || !bytes.Equal(expectedXML, []byte(tc.inputXML)) {
-			t.Fatalf("Test case %d: Expected bucket encryption XML %s but got %s", i+1, string(expectedXML), tc.inputXML)
+			t.Errorf("Test case %d: Expected bucket encryption XML %s but got %s", i+1, string(expectedXML), tc.inputXML)
 		}
 	}
 }

internal/config/config.go

@@ -70,46 +70,46 @@ const (
 
 // Top level config constants.
 const (
-	CredentialsSubSys    = "credentials"
-	PolicyOPASubSys      = "policy_opa"
-	PolicyPluginSubSys   = "policy_plugin"
-	IdentityOpenIDSubSys = "identity_openid"
-	IdentityLDAPSubSys   = "identity_ldap"
-	IdentityTLSSubSys    = "identity_tls"
-	IdentityPluginSubSys = "identity_plugin"
-	CacheSubSys          = "cache"
-	SiteSubSys           = "site"
-	RegionSubSys         = "region"
-	EtcdSubSys           = "etcd"
-	StorageClassSubSys   = "storage_class"
-	APISubSys            = "api"
-	CompressionSubSys    = "compression"
-	LoggerWebhookSubSys  = "logger_webhook"
-	AuditWebhookSubSys   = "audit_webhook"
-	AuditKafkaSubSys     = "audit_kafka"
-	HealSubSys           = "heal"
-	ScannerSubSys        = "scanner"
-	CrawlerSubSys        = "crawler"
-	SubnetSubSys         = "subnet"
-	CallhomeSubSys       = "callhome"
+	CredentialsSubSys    = madmin.CredentialsSubSys
+	PolicyOPASubSys      = madmin.PolicyOPASubSys
+	PolicyPluginSubSys   = madmin.PolicyPluginSubSys
+	IdentityOpenIDSubSys = madmin.IdentityOpenIDSubSys
+	IdentityLDAPSubSys   = madmin.IdentityLDAPSubSys
+	IdentityTLSSubSys    = madmin.IdentityTLSSubSys
+	IdentityPluginSubSys = madmin.IdentityPluginSubSys
+	CacheSubSys          = madmin.CacheSubSys
+	SiteSubSys           = madmin.SiteSubSys
+	RegionSubSys         = madmin.RegionSubSys
+	EtcdSubSys           = madmin.EtcdSubSys
+	StorageClassSubSys   = madmin.StorageClassSubSys
+	APISubSys            = madmin.APISubSys
+	CompressionSubSys    = madmin.CompressionSubSys
+	LoggerWebhookSubSys  = madmin.LoggerWebhookSubSys
+	AuditWebhookSubSys   = madmin.AuditWebhookSubSys
+	AuditKafkaSubSys     = madmin.AuditKafkaSubSys
+	HealSubSys           = madmin.HealSubSys
+	ScannerSubSys        = madmin.ScannerSubSys
+	CrawlerSubSys        = madmin.CrawlerSubSys
+	SubnetSubSys         = madmin.SubnetSubSys
+	CallhomeSubSys       = madmin.CallhomeSubSys
 
-	// Add new constants here if you add new fields to config.
+	// Add new constants here (similar to above) if you add new fields to config.
 )
 
 // Notification config constants.
 const (
-	NotifyKafkaSubSys    = "notify_kafka"
-	NotifyMQTTSubSys     = "notify_mqtt"
-	NotifyMySQLSubSys    = "notify_mysql"
-	NotifyNATSSubSys     = "notify_nats"
-	NotifyNSQSubSys      = "notify_nsq"
-	NotifyESSubSys       = "notify_elasticsearch"
-	NotifyAMQPSubSys     = "notify_amqp"
-	NotifyPostgresSubSys = "notify_postgres"
-	NotifyRedisSubSys    = "notify_redis"
-	NotifyWebhookSubSys  = "notify_webhook"
+	NotifyKafkaSubSys    = madmin.NotifyKafkaSubSys
+	NotifyMQTTSubSys     = madmin.NotifyMQTTSubSys
+	NotifyMySQLSubSys    = madmin.NotifyMySQLSubSys
+	NotifyNATSSubSys     = madmin.NotifyNATSSubSys
+	NotifyNSQSubSys      = madmin.NotifyNSQSubSys
+	NotifyESSubSys       = madmin.NotifyESSubSys
+	NotifyAMQPSubSys     = madmin.NotifyAMQPSubSys
+	NotifyPostgresSubSys = madmin.NotifyPostgresSubSys
+	NotifyRedisSubSys    = madmin.NotifyRedisSubSys
+	NotifyWebhookSubSys  = madmin.NotifyWebhookSubSys
 
-	// Add new constants here if you add new fields to config.
+	// Add new constants here (similar to above) if you add new fields to config.
 )
 
 // NotifySubSystems - all notification sub-systems
@@ -134,39 +134,7 @@ var LoggerSubSystems = set.CreateStringSet(
 )
 
 // SubSystems - all supported sub-systems
-var SubSystems = set.CreateStringSet(
-	CredentialsSubSys,
-	SiteSubSys,
-	RegionSubSys,
-	EtcdSubSys,
-	CacheSubSys,
-	APISubSys,
-	StorageClassSubSys,
-	CompressionSubSys,
-	LoggerWebhookSubSys,
-	AuditWebhookSubSys,
-	AuditKafkaSubSys,
-	PolicyOPASubSys,
-	PolicyPluginSubSys,
-	IdentityLDAPSubSys,
-	IdentityOpenIDSubSys,
-	IdentityTLSSubSys,
-	IdentityPluginSubSys,
-	ScannerSubSys,
-	HealSubSys,
-	NotifyAMQPSubSys,
-	NotifyESSubSys,
-	NotifyKafkaSubSys,
-	NotifyMQTTSubSys,
-	NotifyMySQLSubSys,
-	NotifyNATSSubSys,
-	NotifyNSQSubSys,
-	NotifyPostgresSubSys,
-	NotifyRedisSubSys,
-	NotifyWebhookSubSys,
-	SubnetSubSys,
-	CallhomeSubSys,
-)
+var SubSystems = madmin.SubSystems
 
 // SubSystemsDynamic - all sub-systems that have dynamic config.
 var SubSystemsDynamic = set.CreateStringSet(
@@ -183,7 +151,7 @@ var SubSystemsDynamic = set.CreateStringSet(
 )
 
 // SubSystemsSingleTargets - subsystems which only support single target.
-var SubSystemsSingleTargets = set.CreateStringSet([]string{
+var SubSystemsSingleTargets = set.CreateStringSet(
 	CredentialsSubSys,
 	SiteSubSys,
 	RegionSubSys,
@@ -199,7 +167,9 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	IdentityPluginSubSys,
 	HealSubSys,
 	ScannerSubSys,
-}...)
+	SubnetSubSys,
+	CallhomeSubSys,
+)
 
 // Constant separators
 const (
@@ -212,8 +182,8 @@ const (
 	KvSingleQuote      = madmin.KvSingleQuote
 
 	// Env prefix used for all envs in MinIO
-	EnvPrefix        = "MINIO_"
-	EnvWordDelimiter = `_`
+	EnvPrefix        = madmin.EnvPrefix
+	EnvWordDelimiter = madmin.EnvWordDelimiter
 )
 
 // DefaultKVS - default kvs for all sub-systems
@@ -850,6 +820,33 @@ func GetSubSys(s string) (subSys string, inputs []string, tgt string, e error) {
 	return subSys, inputs, tgt, e
 }
 
+// kvFields - converts an input string of form "k1=v1 k2=v2" into
+// fields of ["k1=v1", "k2=v2"], the tokenization of each `k=v`
+// happens with the right number of input keys, if keys
+// input is empty returned value is empty slice as well.
+func kvFields(input string, keys []string) []string {
+	valueIndexes := make([]int, 0, len(keys))
+	for _, key := range keys {
+		i := strings.Index(input, key+KvSeparator)
+		if i == -1 {
+			continue
+		}
+		valueIndexes = append(valueIndexes, i)
+	}
+
+	sort.Ints(valueIndexes)
+	fields := make([]string, len(valueIndexes))
+	for i := range valueIndexes {
+		j := i + 1
+		if j < len(valueIndexes) {
+			fields[i] = strings.TrimSpace(input[valueIndexes[i]:valueIndexes[j]])
+		} else {
+			fields[i] = strings.TrimSpace(input[valueIndexes[i]:])
+		}
+	}
+	return fields
+}
+
 // SetKVS - set specific key values per sub-system.
 func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err error) {
 	subSys, inputs, tgt, err := GetSubSys(s)
@@ -859,7 +856,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err e
 
 	dynamic = SubSystemsDynamic.Contains(subSys)
 
-	fields := madmin.KvFields(inputs[1], defaultKVS[subSys].Keys())
+	fields := kvFields(inputs[1], defaultKVS[subSys].Keys())
 	if len(fields) == 0 {
 		return false, Errorf("sub-system '%s' cannot have empty keys", subSys)
 	}
@@ -951,7 +948,7 @@ func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err e
 func (c Config) CheckValidKeys(subSys string, deprecatedKeys []string) error {
 	defKVS, ok := DefaultKVS[subSys]
 	if !ok {
-		return fmt.Errorf("Subsystem %s does not exist", subSys)
+		return Errorf("Subsystem %s does not exist", subSys)
 	}
 
 	// Make a list of valid keys for the subsystem including the `comment`
@@ -976,7 +973,7 @@ func (c Config) CheckValidKeys(subSys string, deprecatedKeys []string) error {
 
 	isSingleTarget := SubSystemsSingleTargets.Contains(subSys)
 	if isSingleTarget && len(candidates) > 0 {
-		return fmt.Errorf("The following environment variables are unknown: %s",
+		return Errorf("The following environment variables are unknown: %s",
 			strings.Join(candidates.ToSlice(), ", "))
 	}
 
@@ -996,7 +993,7 @@ func (c Config) CheckValidKeys(subSys string, deprecatedKeys []string) error {
 
 		// Whatever remains are invalid env vars - return an error.
 		if len(candidates) > 0 {
-			return fmt.Errorf("The following environment variables are unknown: %s",
+			return Errorf("The following environment variables are unknown: %s",
 				strings.Join(candidates.ToSlice(), ", "))
 		}
 	}
@@ -1032,7 +1029,7 @@ func (c Config) GetAvailableTargets(subSys string) ([]string, error) {
 
 	defKVS, ok := DefaultKVS[subSys]
 	if !ok {
-		return nil, fmt.Errorf("Subsystem %s does not exist", subSys)
+		return nil, Errorf("Subsystem %s does not exist", subSys)
 	}
 
 	kvsMap := c[subSys]
@@ -1163,13 +1160,13 @@ type KVSrc struct {
 func (c Config) GetResolvedConfigParams(subSys, target string) ([]KVSrc, error) {
 	// Initially only support OpenID
 	if !resolvableSubsystems.Contains(subSys) {
-		return nil, fmt.Errorf("unsupported subsystem: %s", subSys)
+		return nil, Errorf("unsupported subsystem: %s", subSys)
 	}
 
 	// Check if config param requested is valid.
 	defKVS, ok := DefaultKVS[subSys]
 	if !ok {
-		return nil, fmt.Errorf("unknown subsystem: %s", subSys)
+		return nil, Errorf("unknown subsystem: %s", subSys)
 	}
 
 	r := make([]KVSrc, 0, len(defKVS)+1)
@@ -1217,18 +1214,21 @@ type EnvPair struct {
 // SubsysInfo holds config info for a subsystem target.
 type SubsysInfo struct {
 	SubSys, Target string
-	Params         KVS
+	Defaults       KVS
+	Config         KVS
 
 	// map of config parameter name to EnvPair.
 	EnvMap map[string]EnvPair
 }
 
-// GetSubsysInfo returns `SubsysInfo`s for all targets for the subsystem.
-func (c Config) GetSubsysInfo(subSys string) ([]SubsysInfo, error) {
+// GetSubsysInfo returns `SubsysInfo`s for all targets for the subsystem, when
+// target is empty. Otherwise returns `SubsysInfo` for the desired target only.
+// To request the default target only, target must be set to `Default`.
+func (c Config) GetSubsysInfo(subSys, target string) ([]SubsysInfo, error) {
 	// Check if config param requested is valid.
 	defKVS1, ok := DefaultKVS[subSys]
 	if !ok {
-		return nil, fmt.Errorf("unknown subsystem: %s", subSys)
+		return nil, Errorf("unknown subsystem: %s", subSys)
 	}
 
 	targets, err := c.GetAvailableTargets(subSys)
@@ -1236,6 +1236,20 @@ func (c Config) GetSubsysInfo(subSys string) ([]SubsysInfo, error) {
 		return nil, err
 	}
 
+	if target != "" {
+		found := false
+		for _, t := range targets {
+			if t == target {
+				found = true
+				break
+			}
+		}
+		if !found {
+			return nil, Errorf("there is no target `%s` for subsystem `%s`", target, subSys)
+		}
+		targets = []string{target}
+	}
+
 	// The `Comment` configuration variable is optional but is available to be
 	// set for all sub-systems. It is not present in the `DefaultKVS` map's
 	// values. To enable fetching a configured comment value from the
@@ -1248,10 +1262,11 @@ func (c Config) GetSubsysInfo(subSys string) ([]SubsysInfo, error) {
 	for _, target := range targets {
 		kvs := c.getTargetKVS(subSys, target)
 		cs := SubsysInfo{
-			SubSys: subSys,
-			Target: target,
-			Params: kvs,
-			EnvMap: make(map[string]EnvPair),
+			SubSys:   subSys,
+			Target:   target,
+			Defaults: defKVS,
+			Config:   kvs,
+			EnvMap:   make(map[string]EnvPair),
 		}
 
 		// Add all env vars that are set.
@@ -1274,7 +1289,7 @@ func (c Config) GetSubsysInfo(subSys string) ([]SubsysInfo, error) {
 
 // AddEnvString adds env vars to the given string builder.
 func (cs *SubsysInfo) AddEnvString(b *strings.Builder) {
-	for _, v := range cs.Params {
+	for _, v := range cs.Defaults {
 		if ep, ok := cs.EnvMap[v.Key]; ok {
 			b.WriteString(KvComment)
 			b.WriteString(KvSpaceSeparator)
@@ -1301,6 +1316,6 @@ func (cs *SubsysInfo) AddString(b *strings.Builder, off bool) {
 		b.WriteString(cs.Target)
 	}
 	b.WriteString(KvSpaceSeparator)
-	b.WriteString(cs.Params.String())
+	b.WriteString(cs.Config.String())
 	b.WriteString(KvNewline)
 }

internal/config/config_test.go

@@ -19,8 +19,6 @@ package config
 
 import (
 	"testing"
-
-	"github.com/minio/madmin-go"
 )
 
 func TestKVFields(t *testing.T) {
@@ -92,7 +90,7 @@ func TestKVFields(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run("", func(t *testing.T) {
-			gotFields := madmin.KvFields(test.input, test.keys)
+			gotFields := kvFields(test.input, test.keys)
 			if len(gotFields) != len(test.expectedFields) {
 				t.Errorf("Expected keys %d, found %d", len(test.expectedFields), len(gotFields))
 			}

internal/config/identity/ldap/ldap.go

@@ -208,25 +208,29 @@ func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
 		l.ServerAddr = net.JoinHostPort(l.ServerAddr, "636")
 	}
 
-	if l.serverInsecure {
-		return ldap.Dial("tcp", l.ServerAddr)
-	}
-
 	tlsConfig := &tls.Config{
 		InsecureSkipVerify: l.tlsSkipVerify,
 		RootCAs:            l.rootCAs,
 	}
 
-	if l.serverStartTLS {
-		conn, err := ldap.Dial("tcp", l.ServerAddr)
-		if err != nil {
-			return nil, err
+	if l.serverInsecure {
+		ldapConn, err = ldap.Dial("tcp", l.ServerAddr)
+	} else {
+		if l.serverStartTLS {
+			ldapConn, err = ldap.Dial("tcp", l.ServerAddr)
+		} else {
+			ldapConn, err = ldap.DialTLS("tcp", l.ServerAddr, tlsConfig)
 		}
-		err = conn.StartTLS(tlsConfig)
-		return conn, err
 	}
 
-	return ldap.DialTLS("tcp", l.ServerAddr, tlsConfig)
+	if ldapConn != nil {
+		ldapConn.SetTimeout(30 * time.Second) // Change default timeout to 30 seconds.
+		if l.serverStartTLS {
+			err = ldapConn.StartTLS(tlsConfig)
+		}
+	}
+
+	return ldapConn, err
 }
 
 // GetExpiryDuration - return parsed expiry duration.

internal/config/scanner/scanner.go

@@ -52,11 +52,11 @@ type Config struct {
 var DefaultKVS = config.KVS{
 	config.KV{
 		Key:   Delay,
-		Value: "10",
+		Value: "2",
 	},
 	config.KV{
 		Key:   MaxWait,
-		Value: "15s",
+		Value: "5s",
 	},
 	config.KV{
 		Key:   Cycle,

internal/config/subnet/config.go

@@ -55,7 +55,7 @@ type Config struct {
 	ProxyURL *xnet.URL `json:"proxy_url"`
 
 	// Transport configured with proxy_url if set optionally.
-	transport *http.Transport
+	transport http.RoundTripper
 }
 
 // LookupConfig - lookup config and override with valid environment settings if any.
@@ -83,11 +83,13 @@ func LookupConfig(kvs config.KVS, transport http.RoundTripper) (cfg Config, err
 	}
 
 	// Make sure to clone the transport before editing the ProxyURL
-	ctransport := transport.(*http.Transport).Clone()
 	if cfg.ProxyURL != nil {
+		ctransport := transport.(*http.Transport).Clone()
 		ctransport.Proxy = http.ProxyURL((*url.URL)(cfg.ProxyURL))
+		cfg.transport = ctransport
+	} else {
+		cfg.transport = transport
 	}
-	cfg.transport = ctransport
 
 	return cfg, nil
 }

internal/crypto/error.go

@@ -76,6 +76,9 @@ var (
 	// ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible
 	// The client needs to remove the SSE-S3 header or the SSE-C headers
 	ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers")
+
+	// ErrInvalidEncryptionKeyID returns error when KMS key id contains invalid characters
+	ErrInvalidEncryptionKeyID = Errorf("KMS KeyID contains unsupported characters")
 )
 
 var (

internal/crypto/metadata.go

@@ -56,6 +56,9 @@ const (
 	// be part of the object. Therefore, the bucket/object name must be added
 	// to the context, if not present, whenever a decryption is performed.
 	MetaContext = "X-Minio-Internal-Server-Side-Encryption-Context"
+
+	// ARNPrefix prefix for "arn:aws:kms"
+	ARNPrefix = "arn:aws:kms:"
 )
 
 // IsMultiPart returns true if the object metadata indicates

internal/crypto/sse-kms.go

@@ -55,7 +55,8 @@ func (ssekms) IsRequested(h http.Header) bool {
 		return true
 	}
 	if _, ok := h[xhttp.AmzServerSideEncryption]; ok {
-		return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES // Return only true if the SSE header is specified and does not contain the SSE-S3 value
+		// Return only true if the SSE header is specified and does not contain the SSE-S3 value
+		return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES
 	}
 	return false
 }
@@ -63,6 +64,10 @@ func (ssekms) IsRequested(h http.Header) bool {
 // ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
 // and the KMS context on success.
 func (ssekms) ParseHTTP(h http.Header) (string, kms.Context, error) {
+	if h == nil {
+		return "", nil, ErrInvalidEncryptionMethod
+	}
+
 	algorithm := h.Get(xhttp.AmzServerSideEncryption)
 	if algorithm != xhttp.AmzEncryptionKMS {
 		return "", nil, ErrInvalidEncryptionMethod
@@ -80,7 +85,13 @@ func (ssekms) ParseHTTP(h http.Header) (string, kms.Context, error) {
 			return "", nil, err
 		}
 	}
-	return h.Get(xhttp.AmzServerSideEncryptionKmsID), ctx, nil
+
+	keyID := h.Get(xhttp.AmzServerSideEncryptionKmsID)
+	spaces := strings.HasPrefix(keyID, " ") || strings.HasSuffix(keyID, " ")
+	if spaces {
+		return "", nil, ErrInvalidEncryptionKeyID
+	}
+	return strings.TrimPrefix(keyID, ARNPrefix), ctx, nil
 }
 
 // IsEncrypted returns true if the object metadata indicates

internal/disk/disk.go

@@ -30,4 +30,36 @@ type Info struct {
 	Files  uint64
 	Ffree  uint64
 	FSType string
+	Major  uint32
+	Minor  uint32
+}
+
+// DevID is the drive major and minor ids
+type DevID struct {
+	Major uint32
+	Minor uint32
+}
+
+// AllDrivesIOStats is map between drive devices and IO stats
+type AllDrivesIOStats map[DevID]IOStats
+
+// IOStats contains stats of a single drive
+type IOStats struct {
+	ReadIOs        uint64
+	ReadMerges     uint64
+	ReadSectors    uint64
+	ReadTicks      uint64
+	WriteIOs       uint64
+	WriteMerges    uint64
+	WriteSectors   uint64
+	WriteTicks     uint64
+	CurrentIOs     uint64
+	TotalTicks     uint64
+	ReqTicks       uint64
+	DiscardIOs     uint64
+	DiscardMerges  uint64
+	DiscardSectors uint64
+	DiscardTicks   uint64
+	FlushIOs       uint64
+	FlushTicks     uint64
 }

internal/disk/stat_bsd.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 	"syscall"
 )
@@ -46,3 +47,8 @@ func GetInfo(path string) (info Info, err error) {
 	info.Used = info.Total - info.Free
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/disk/stat_freebsd.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 	"syscall"
 )
@@ -46,3 +47,8 @@ func GetInfo(path string) (info Info, err error) {
 	info.Used = info.Total - info.Free
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/disk/stat_linux.go

@@ -21,8 +21,14 @@
 package disk
 
 import (
+	"bufio"
 	"fmt"
+	"os"
+	"strconv"
+	"strings"
 	"syscall"
+
+	"golang.org/x/sys/unix"
 )
 
 // GetInfo returns total and free bytes available in a directory, e.g. `/`.
@@ -49,5 +55,138 @@ func GetInfo(path string) (info Info, err error) {
 		return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path)
 	}
 	info.Used = info.Total - info.Free
+
+	st := syscall.Stat_t{}
+	err = syscall.Stat(path, &st)
+	if err != nil {
+		return Info{}, err
+	}
+	//nolint:unconvert
+	devID := uint64(st.Dev) // Needed to support multiple GOARCHs
+	info.Major = unix.Major(devID)
+	info.Minor = unix.Minor(devID)
 	return info, nil
 }
+
+const (
+	statsPath = "/proc/diskstats"
+)
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	proc, err := os.Open(statsPath)
+	if err != nil {
+		return nil, err
+	}
+	defer proc.Close()
+
+	ret := make(AllDrivesIOStats)
+
+	sc := bufio.NewScanner(proc)
+	for sc.Scan() {
+		line := sc.Text()
+		fields := strings.Fields(line)
+		if len(fields) < 11 {
+			continue
+		}
+
+		var err error
+		var ds IOStats
+
+		ds.ReadIOs, err = strconv.ParseUint((fields[3]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+		ds.ReadMerges, err = strconv.ParseUint((fields[4]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+		ds.ReadSectors, err = strconv.ParseUint((fields[5]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+		ds.ReadTicks, err = strconv.ParseUint((fields[6]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+		ds.WriteIOs, err = strconv.ParseUint((fields[7]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+		ds.WriteMerges, err = strconv.ParseUint((fields[8]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+		ds.WriteSectors, err = strconv.ParseUint((fields[9]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+		ds.WriteTicks, err = strconv.ParseUint((fields[10]), 10, 64)
+		if err != nil {
+			return ret, err
+		}
+
+		if len(fields) > 11 {
+			ds.CurrentIOs, err = strconv.ParseUint((fields[11]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+
+			ds.TotalTicks, err = strconv.ParseUint((fields[12]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+			ds.ReqTicks, err = strconv.ParseUint((fields[13]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+		}
+
+		if len(fields) > 14 {
+			ds.DiscardIOs, err = strconv.ParseUint((fields[14]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+			ds.DiscardMerges, err = strconv.ParseUint((fields[15]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+			ds.DiscardSectors, err = strconv.ParseUint((fields[16]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+			ds.DiscardTicks, err = strconv.ParseUint((fields[17]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+		}
+
+		if len(fields) > 18 {
+			ds.FlushIOs, err = strconv.ParseUint((fields[18]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+			ds.FlushTicks, err = strconv.ParseUint((fields[19]), 10, 64)
+			if err != nil {
+				return ret, err
+			}
+		}
+
+		major, err := strconv.ParseUint((fields[0]), 10, 32)
+		if err != nil {
+			return ret, err
+		}
+
+		minor, err := strconv.ParseUint((fields[1]), 10, 32)
+		if err != nil {
+			return ret, err
+		}
+		ret[DevID{uint32(major), uint32(minor)}] = ds
+	}
+
+	if err := sc.Err(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}

internal/disk/stat_linux_32bit.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 	"strconv"
 	"syscall"
@@ -81,3 +82,8 @@ func GetInfo(path string) (info Info, err error) {
 	info.Used = info.Total - info.Free
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/disk/stat_linux_s390x.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 	"strconv"
 	"syscall"
@@ -81,3 +82,8 @@ func GetInfo(path string) (info Info, err error) {
 	info.Used = info.Total - info.Free
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/disk/stat_netbsd.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 
 	"golang.org/x/sys/unix"
@@ -46,3 +47,8 @@ func GetInfo(path string) (info Info, err error) {
 	info.Used = info.Total - info.Free
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/disk/stat_openbsd.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 	"syscall"
 )
@@ -46,3 +47,8 @@ func GetInfo(path string) (info Info, err error) {
 	info.Used = info.Total - info.Free
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/disk/stat_solaris.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 
 	"golang.org/x/sys/unix"
@@ -46,3 +47,8 @@ func GetInfo(path string) (info Info, err error) {
 	info.Used = info.Total - info.Free
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/disk/stat_windows.go

@@ -21,6 +21,7 @@
 package disk
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"syscall"
@@ -106,3 +107,8 @@ func GetInfo(path string) (info Info, err error) {
 
 	return info, nil
 }
+
+// GetAllDrivesIOStats returns IO stats of all drives found in the machine
+func GetAllDrivesIOStats() (info AllDrivesIOStats, err error) {
+	return nil, errors.New("operation unsupported")
+}

internal/dsync/drwmutex.go

@@ -59,7 +59,8 @@ const (
 	// dRWMutexRefreshInterval - default the interval between two refresh calls
 	drwMutexRefreshInterval = 10 * time.Second
 
-	lockRetryInterval = 1 * time.Second
+	// maximum time to sleep before retrying a failed blocking lock()
+	lockRetryInterval = 50 * time.Millisecond
 
 	drwMutexInfinite = 1<<63 - 1
 )
@@ -142,7 +143,8 @@ func (dm *DRWMutex) Lock(id, source string) {
 
 // Options lock options.
 type Options struct {
-	Timeout time.Duration
+	Timeout       time.Duration
+	RetryInterval time.Duration
 }
 
 // GetLock tries to get a write lock on dm before the timeout elapses.
@@ -236,7 +238,11 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, lockLossCallback func(), i
 				return locked
 			}
 
-			time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryInterval)))
+			lockRetryInterval := dm.lockRetryInterval
+			if opts.RetryInterval > 0 {
+				lockRetryInterval = opts.RetryInterval
+			}
+			time.Sleep(time.Duration(dm.rng.Float64() * float64(lockRetryInterval)))
 		}
 	}
 }

internal/kernel/kernel.go

@@ -1,130 +0,0 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-//go:build linux
-// +build linux
-
-package kernel
-
-import (
-	"fmt"
-	"io/ioutil"
-	"regexp"
-	"strconv"
-	"strings"
-	"syscall"
-)
-
-var versionRegex = regexp.MustCompile(`^(\d+)\.(\d+).(\d+).*$`)
-
-// VersionFromRelease converts a release string with format
-// 4.4.2[-1] to a kernel version number in LINUX_VERSION_CODE format.
-// That is, for kernel "a.b.c", the version number will be (a<<16 + b<<8 + c)
-func VersionFromRelease(releaseString string) (uint32, error) {
-	versionParts := versionRegex.FindStringSubmatch(releaseString)
-	if len(versionParts) != 4 {
-		return 0, fmt.Errorf("got invalid release version %q (expected format '4.3.2-1')", releaseString)
-	}
-	major, err := strconv.Atoi(versionParts[1])
-	if err != nil {
-		return 0, err
-	}
-
-	minor, err := strconv.Atoi(versionParts[2])
-	if err != nil {
-		return 0, err
-	}
-
-	patch, err := strconv.Atoi(versionParts[3])
-	if err != nil {
-		return 0, err
-	}
-	return Version(major, minor, patch), nil
-}
-
-// Version implements KERNEL_VERSION equivalent macro
-// #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
-func Version(major, minor, patch int) uint32 {
-	if patch > 255 {
-		patch = 255
-	}
-	out := major<<16 + minor<<8 + patch
-	return uint32(out)
-}
-
-func currentVersionUname() (uint32, error) {
-	var buf syscall.Utsname
-	if err := syscall.Uname(&buf); err != nil {
-		return 0, err
-	}
-	releaseString := strings.Trim(utsnameStr(buf.Release[:]), "\x00")
-	return VersionFromRelease(releaseString)
-}
-
-func currentVersionUbuntu() (uint32, error) {
-	procVersion, err := ioutil.ReadFile("/proc/version_signature")
-	if err != nil {
-		return 0, err
-	}
-	var u1, u2, releaseString string
-	_, err = fmt.Sscanf(string(procVersion), "%s %s %s", &u1, &u2, &releaseString)
-	if err != nil {
-		return 0, err
-	}
-	return VersionFromRelease(releaseString)
-}
-
-var debianVersionRegex = regexp.MustCompile(`.* SMP Debian (\d+\.\d+.\d+-\d+)(?:\+[[:alnum:]]*)?.*`)
-
-func parseDebianVersion(str string) (uint32, error) {
-	match := debianVersionRegex.FindStringSubmatch(str)
-	if len(match) != 2 {
-		return 0, fmt.Errorf("failed to parse kernel version from /proc/version: %s", str)
-	}
-	return VersionFromRelease(match[1])
-}
-
-func currentVersionDebian() (uint32, error) {
-	procVersion, err := ioutil.ReadFile("/proc/version")
-	if err != nil {
-		return 0, fmt.Errorf("error reading /proc/version: %s", err)
-	}
-
-	return parseDebianVersion(string(procVersion))
-}
-
-// CurrentVersion returns the current kernel version in
-// LINUX_VERSION_CODE format (see VersionFromRelease())
-func CurrentVersion() (uint32, error) {
-	// We need extra checks for Debian and Ubuntu as they modify
-	// the kernel version patch number for compatibility with
-	// out-of-tree modules. Linux perf tools do the same for Ubuntu
-	// systems: https://github.com/torvalds/linux/commit/d18acd15c
-	//
-	// See also:
-	// https://kernel-team.pages.debian.net/kernel-handbook/ch-versions.html
-	// https://wiki.ubuntu.com/Kernel/FAQ
-	version, err := currentVersionUbuntu()
-	if err == nil {
-		return version, nil
-	}
-	version, err = currentVersionDebian()
-	if err == nil {
-		return version, nil
-	}
-	return currentVersionUname()
-}

internal/kernel/kernel_other.go

@@ -1,36 +0,0 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-//go:build !linux
-// +build !linux
-
-package kernel
-
-// VersionFromRelease only implemented on Linux.
-func VersionFromRelease(_ string) (uint32, error) {
-	return 0, nil
-}
-
-// Version only implemented on Linux.
-func Version(_, _, _ int) uint32 {
-	return 0
-}
-
-// CurrentVersion only implemented on Linux.
-func CurrentVersion() (uint32, error) {
-	return 0, nil
-}

internal/kernel/kernel_test.go

@@ -1,88 +0,0 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-//go:build linux
-// +build linux
-
-package kernel
-
-import "testing"
-
-var testData = []struct {
-	success       bool
-	releaseString string
-	kernelVersion uint32
-}{
-	{true, "4.1.2-3", 262402},
-	{true, "4.8.14-200.fc24.x86_64", 264206},
-	{true, "4.1.2-3foo", 262402},
-	{true, "4.1.2foo-1", 262402},
-	{true, "4.1.2-rkt-v1", 262402},
-	{true, "4.1.2rkt-v1", 262402},
-	{true, "4.1.2-3 foo", 262402},
-	{true, "3.10.0-1062.el7.x86_64", 199168},
-	{true, "3.0.0", 196608},
-	{true, "2.6.32", 132640},
-	{true, "5.13.0-30-generic", 331008},
-	{true, "5.10.0-1052-oem", 330240},
-	{false, "foo 4.1.2-3", 0},
-	{true, "4.1.2", 262402},
-	{false, ".4.1.2", 0},
-	{false, "4.1.", 0},
-	{false, "4.1", 0},
-}
-
-func TestVersionFromRelease(t *testing.T) {
-	for _, test := range testData {
-		version, err := VersionFromRelease(test.releaseString)
-		if err != nil && test.success {
-			t.Errorf("expected %q to success: %s", test.releaseString, err)
-		} else if err == nil && !test.success {
-			t.Errorf("expected %q to fail", test.releaseString)
-		}
-		if version != test.kernelVersion {
-			t.Errorf("expected kernel version %d, got %d", test.kernelVersion, version)
-		}
-	}
-}
-
-func TestParseDebianVersion(t *testing.T) {
-	for _, tc := range []struct {
-		success       bool
-		releaseString string
-		kernelVersion uint32
-	}{
-		// 4.9.168
-		{true, "Linux version 4.9.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.168-1+deb9u3 (2019-06-16)", 264616},
-		// 4.9.88
-		{true, "Linux ip-10-0-75-49 4.9.0-6-amd64 #1 SMP Debian 4.9.88-1+deb9u1 (2018-05-07) x86_64 GNU/Linux", 264536},
-		// 3.0.4
-		{true, "Linux version 3.16.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 4.9.2 (Debian 4.9.2-10+deb8u2) ) #1 SMP Debian 3.16.68-1 (2019-05-22)", 200772},
-		// Invalid
-		{false, "Linux version 4.9.125-linuxkit (root@659b6d51c354) (gcc version 6.4.0 (Alpine 6.4.0) ) #1 SMP Fri Sep 7 08:20:28 UTC 2018", 0},
-	} {
-		version, err := parseDebianVersion(tc.releaseString)
-		if err != nil && tc.success {
-			t.Errorf("expected %q to success: %s", tc.releaseString, err)
-		} else if err == nil && !tc.success {
-			t.Errorf("expected %q to fail", tc.releaseString)
-		}
-		if version != tc.kernelVersion {
-			t.Errorf("expected kernel version %d, got %d", tc.kernelVersion, version)
-		}
-	}
-}

internal/kernel/kernel_utsname_int8.go

@@ -1,32 +0,0 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-//go:build (linux && 386) || (linux && amd64) || (linux && arm64) || (linux && mips64) || (linux && mips)
-// +build linux,386 linux,amd64 linux,arm64 linux,mips64 linux,mips
-
-package kernel
-
-func utsnameStr(in []int8) string {
-	out := make([]byte, 0, len(in))
-	for i := 0; i < len(in); i++ {
-		if in[i] == 0x00 {
-			break
-		}
-		out = append(out, byte(in[i]))
-	}
-	return string(out)
-}