Update console to 0.22.5 (#16400)

Co-authored-by: Anis Elleuch <anis@min.io>

cmd/admin-handlers-site-replication.go

@@ -114,8 +114,7 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 	default:
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.MakeWithVersioningBktOp:
-		createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
-		createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
+		createdAt, cerr := time.Parse(time.RFC3339Nano, strings.TrimSpace(r.Form.Get("createdAt")))
 		if cerr != nil {
 			createdAt = timeSentinel
 		}
@@ -132,7 +131,6 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 	case madmin.DeleteBucketBktOp, madmin.ForceDeleteBucketBktOp:
 		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
 			Force:      operation == madmin.ForceDeleteBucketBktOp,
-			NoRecreate: r.Form.Get("noRecreate") == "true",
 			SRDeleteOp: getSRBucketDeleteOp(true),
 		})
 	case madmin.PurgeDeletedBucketOp:

cmd/admin-handlers-users.go

@@ -665,6 +665,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	opts := newServiceAccountOpts{
 		accessKey: createReq.AccessKey,
 		secretKey: createReq.SecretKey,
+		comment:   createReq.Comment,
 		claims:    make(map[string]interface{}),
 	}
 
@@ -805,6 +806,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 					AccessKey:     newCred.AccessKey,
 					SecretKey:     newCred.SecretKey,
 					Groups:        newCred.Groups,
+					Comment:       newCred.Comment,
 					Claims:        opts.claims,
 					SessionPolicy: createReq.Policy,
 					Status:        auth.AccountOn,
@@ -892,6 +894,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 	opts := updateServiceAccountOpts{
 		secretKey:     updateReq.NewSecretKey,
 		status:        updateReq.NewStatus,
+		comment:       updateReq.NewComment,
 		sessionPolicy: sp,
 	}
 	updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts)
@@ -909,6 +912,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 					AccessKey:     accessKey,
 					SecretKey:     opts.secretKey,
 					Status:        opts.status,
+					Comment:       opts.comment,
 					SessionPolicy: updateReq.NewPolicy,
 				},
 			},
@@ -993,6 +997,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 
 	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
+		Comment:       svcAccount.Comment,
 		AccountStatus: svcAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
@@ -2467,6 +2472,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					opts := updateServiceAccountOpts{
 						secretKey:     svcAcctReq.SecretKey,
 						status:        svcAcctReq.Status,
+						comment:       svcAcctReq.Comment,
 						sessionPolicy: sp,
 					}
 					_, err = globalIAMSys.UpdateServiceAccount(ctx, svcAcctReq.AccessKey, opts)
@@ -2481,6 +2487,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					secretKey:     svcAcctReq.SecretKey,
 					sessionPolicy: sp,
 					claims:        svcAcctReq.Claims,
+					comment:       svcAcctReq.Comment,
 				}
 
 				// In case of LDAP we need to resolve the targetUser to a DN and

cmd/admin-handlers.go

@@ -1332,7 +1332,6 @@ func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName
 func deleteObjectPerfBucket(objectAPI ObjectLayer) {
 	objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
 		Force:      true,
-		NoRecreate: true,
 		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 	})
 }

cmd/background-newdisks-heal-ops.go

@@ -79,6 +79,10 @@ type healingTracker struct {
 
 	// Filled during heal.
 	HealedBuckets []string
+
+	// ID of the current healing operation
+	HealID string
+
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -112,11 +116,12 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 }
 
 // newHealingTracker will create a new healing tracker for the disk.
-func newHealingTracker(disk StorageAPI) *healingTracker {
+func newHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	diskID, _ := disk.GetDiskID()
 	h := healingTracker{
 		disk:     disk,
 		ID:       diskID,
+		HealID:   healID,
 		Path:     disk.String(),
 		Endpoint: disk.Endpoint().String(),
 		Started:  time.Now().UTC(),
@@ -227,6 +232,7 @@ func (h *healingTracker) printTo(writer io.Writer) {
 func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 	return madmin.HealingDisk{
 		ID:                h.ID,
+		HealID:            h.HealID,
 		Endpoint:          h.Endpoint,
 		PoolIndex:         h.PoolIndex,
 		SetIndex:          h.SetIndex,
@@ -286,8 +292,6 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
 func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
-	logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
-
 	disk, format, err := connectEndpoint(endpoint)
 	if err != nil {
 		return fmt.Errorf("Error: %w, %s", err, endpoint)
@@ -318,6 +322,20 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	ctx = lkctx.Context()
 	defer locker.Unlock(lkctx)
 
+	// Load healing tracker in this disk
+	tracker, err := loadHealingTracker(ctx, disk)
+	if err != nil {
+		// A healing track can be not found when another disk in the same
+		// erasure set and same healing-id successfully finished healing.
+		if err == errFileNotFound {
+			return nil
+		}
+		logger.LogIf(ctx, fmt.Errorf("Unable to load a healing tracker on '%s': %w", disk, err))
+		tracker = newHealingTracker(disk, mustGetUUID())
+	}
+
+	logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
+
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to heal all bucket metadata configuration.
@@ -340,15 +358,6 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
 	}
 
-	// Load healing tracker in this disk
-	tracker, err := loadHealingTracker(ctx, disk)
-	if err != nil {
-		// So someone changed the drives underneath, healing tracker missing.
-		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
-			disk, humanize.Ordinal(poolIdx+1), err))
-		tracker = newHealingTracker(disk)
-	}
-
 	// Load bucket totals
 	cache := dataUsageCache{}
 	if err := cache.load(ctx, z.serverPools[poolIdx].sets[setIdx], dataUsageCacheName); err == nil {
@@ -379,7 +388,24 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		logger.Info("\n")
 	}
 
-	logger.LogIf(ctx, tracker.delete(ctx))
+	if tracker.HealID == "" { // HealID is empty only before Feb 2023
+		logger.LogIf(ctx, tracker.delete(ctx))
+		return nil
+	}
+
+	// Remove .healing.bin from all disks with similar heal-id
+	for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
+		t, err := loadHealingTracker(ctx, disk)
+		if err != nil {
+			if err != errFileNotFound {
+				logger.LogIf(ctx, err)
+			}
+			continue
+		}
+		if t.HealID == tracker.HealID {
+			t.delete(ctx)
+		}
+	}
 
 	return nil
 }

cmd/background-newdisks-heal-ops_gen.go

@@ -182,6 +182,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "HealID":
+			z.HealID, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "HealID")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -195,9 +201,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 22
+	// map header, size 23
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x16, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -430,15 +436,25 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "HealID"
+	err = en.Append(0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.HealID)
+	if err != nil {
+		err = msgp.WrapError(err, "HealID")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 22
+	// map header, size 23
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x16, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -509,6 +525,9 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0002 := range z.HealedBuckets {
 		o = msgp.AppendString(o, z.HealedBuckets[za0002])
 	}
+	// string "HealID"
+	o = append(o, 0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
+	o = msgp.AppendString(o, z.HealID)
 	return
 }
 
@@ -688,6 +707,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "HealID":
+			z.HealID, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "HealID")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -710,5 +735,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
+	s += 7 + msgp.StringPrefixSize + len(z.HealID)
 	return
 }

cmd/bucket-handlers.go

@@ -791,8 +791,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
-						Force:      false,
-						NoRecreate: true,
+						Force:      true,
 						SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 					})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)

cmd/bucket-metadata-sys.go

@@ -66,7 +66,7 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
 // so they should be replaced atomically and not appended to, etc.
 // Data is not persisted to disk.
 func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
-	if bucket != minioMetaBucket {
+	if !isMinioMetaBucketName(bucket) {
 		sys.Lock()
 		sys.metadataMap[bucket] = meta
 		sys.Unlock()
@@ -79,7 +79,7 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		return updatedAt, errServerNotInitialized
 	}
 
-	if bucket == minioMetaBucket {
+	if isMinioMetaBucketName(bucket) {
 		return updatedAt, errInvalidArgument
 	}
 
@@ -164,7 +164,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 // For all other bucket specific metadata, use the relevant
 // calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
-	if bucket == minioMetaBucket {
+	if isMinioMetaBucketName(bucket) {
 		return newBucketMetadata(bucket), errConfigNotFound
 	}
 
@@ -345,7 +345,7 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (Buc
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}
 
-	if bucket == minioMetaBucket {
+	if isMinioMetaBucketName(bucket) {
 		return newBucketMetadata(bucket), errInvalidArgument
 	}
 

cmd/bucket-metadata.go

@@ -135,7 +135,7 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
 	if name == "" {
 		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
-		return errors.New("bucket name cannot be empty")
+		return errInvalidArgument
 	}
 	configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)

cmd/common-main.go

@@ -117,7 +117,8 @@ func init() {
 	go func() {
 		var t *time.Ticker
 		if containers {
-			t = time.NewTicker(1 * time.Minute)
+			// k8s DNS TTL is 30s (Attempt a refresh only after)
+			t = time.NewTicker(30 * time.Second)
 		} else {
 			t = time.NewTicker(10 * time.Minute)
 		}

cmd/endpoint.go

@@ -417,6 +417,27 @@ func (endpoints Endpoints) atleastOneEndpointLocal() bool {
 	return false
 }
 
+func hostResolveToLocalhost(endpoint Endpoint) bool {
+	hostIPs, err := getHostIP(endpoint.Hostname())
+	if err != nil {
+		// Log the message to console about the host resolving
+		reqInfo := (&logger.ReqInfo{}).AppendTags(
+			"host",
+			endpoint.Hostname(),
+		)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+		logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
+		return false
+	}
+	var loopback int
+	for _, hostIP := range hostIPs.ToSlice() {
+		if net.ParseIP(hostIP).IsLoopback() {
+			loopback++
+		}
+	}
+	return loopback == len(hostIPs)
+}
+
 // UpdateIsLocal - resolves the host and discovers the local host.
 func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	orchestrated := IsDocker() || IsKubernetes()
@@ -452,6 +473,26 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 					endpoints[i].Hostname(),
 				)
 
+				if orchestrated && hostResolveToLocalhost(endpoints[i]) {
+					// time elapsed
+					timeElapsed := time.Since(startTime)
+					// log error only if more than a second has elapsed
+					if timeElapsed > time.Second {
+						reqInfo.AppendTags("elapsedTime",
+							humanize.RelTime(startTime,
+								startTime.Add(timeElapsed),
+								"elapsed",
+								"",
+							))
+						ctx := logger.SetReqInfo(GlobalContext,
+							reqInfo)
+						logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
+							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.Application)
+					}
+
+					continue
+				}
+
 				// return err if not Docker or Kubernetes
 				// We use IsDocker() to check for Docker environment
 				// We use IsKubernetes() to check for Kubernetes environment
@@ -465,7 +506,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 				if err != nil {
 					// time elapsed
 					timeElapsed := time.Since(startTime)
-					// log error only if more than 1s elapsed
+					// log error only if more than a second has elapsed
 					if timeElapsed > time.Second {
 						reqInfo.AppendTags("elapsedTime",
 							humanize.RelTime(startTime,
@@ -475,7 +516,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogIf(ctx, err, logger.Application)
+						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.Application)
 					}
 				} else {
 					resolvedList[i] = true

cmd/erasure-bucket.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 )
 
@@ -31,169 +30,6 @@ var bucketOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnform
 // list all errors that can be ignored in a bucket metadata operation.
 var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
 
-// Bucket operations
-
-// MakeBucket - make a bucket.
-func (er erasureObjects) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Make a volume entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] != nil {
-				if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
-					if opts.ForceCreate && errors.Is(err, errVolumeExists) {
-						// No need to return error when force create was
-						// requested.
-						return nil
-					}
-					if !errors.Is(err, errVolumeExists) {
-						logger.LogIf(ctx, err)
-					}
-					return err
-				}
-				return nil
-			}
-			return errDiskNotFound
-		}, index)
-	}
-
-	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
-	return toObjectErr(err, bucket)
-}
-
-func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
-	g := errgroup.WithNErrs(len(storageDisks))
-	// Undo previous make bucket entry on all underlying storage disks.
-	for index := range storageDisks {
-		if storageDisks[index] == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			_ = storageDisks[index].MakeVol(context.Background(), bucket)
-			return nil
-		}, index)
-	}
-
-	// Wait for all make vol to finish.
-	g.Wait()
-}
-
-// getBucketInfo - returns the BucketInfo from one of the load balanced disks.
-func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-	bucketsInfo := make([]BucketInfo, len(storageDisks))
-	// Undo previous make bucket entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] == nil {
-				return errDiskNotFound
-			}
-			volInfo, err := storageDisks[index].StatVol(ctx, bucketName)
-			if err != nil {
-				if opts.Deleted {
-					dvi, derr := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucketName))
-					if derr != nil {
-						return err
-					}
-					bucketsInfo[index] = BucketInfo{Name: bucketName, Deleted: dvi.Created}
-					return nil
-				}
-				return err
-			}
-			bucketsInfo[index] = BucketInfo{Name: volInfo.Name, Created: volInfo.Created}
-			return nil
-		}, index)
-	}
-
-	errs := g.Wait()
-
-	for i, err := range errs {
-		if err == nil {
-			return bucketsInfo[i], nil
-		}
-	}
-
-	// If all our errors were ignored, then we try to
-	// reduce to one error based on read quorum.
-	// `nil` is deliberately passed for ignoredErrs
-	// because these errors were already ignored.
-	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, er.defaultRQuorum())
-}
-
-// GetBucketInfo - returns BucketInfo for a bucket.
-func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
-	bucketInfo, err := er.getBucketInfo(ctx, bucket, opts)
-	if err != nil {
-		return bi, toObjectErr(err, bucket)
-	}
-	return bucketInfo, nil
-}
-
-// DeleteBucket - deletes a bucket.
-func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] != nil {
-				return storageDisks[index].DeleteVol(ctx, bucket, opts.Force)
-			}
-			return errDiskNotFound
-		}, index)
-	}
-
-	// Wait for all the delete vols to finish.
-	dErrs := g.Wait()
-
-	if opts.Force {
-		for _, err := range dErrs {
-			if err != nil {
-				undoDeleteBucket(storageDisks, bucket)
-				return toObjectErr(err, bucket)
-			}
-		}
-
-		return nil
-	}
-
-	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, er.defaultWQuorum())
-	if err == errErasureWriteQuorum && !opts.NoRecreate {
-		undoDeleteBucket(storageDisks, bucket)
-	}
-
-	if err == nil || errors.Is(err, errVolumeNotFound) {
-		var purgedDangling bool
-		// At this point we have `err == nil` but some errors might be `errVolumeNotEmpty`
-		// we should proceed to attempt a force delete of such buckets.
-		for index, err := range dErrs {
-			if err == errVolumeNotEmpty && storageDisks[index] != nil {
-				storageDisks[index].RenameFile(ctx, bucket, "", minioMetaTmpDeletedBucket, mustGetUUID())
-				purgedDangling = true
-			}
-		}
-		// if we purged dangling buckets, ignore errVolumeNotFound error.
-		if purgedDangling {
-			err = nil
-		}
-		if opts.SRDeleteOp == MarkDelete {
-			er.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
-		}
-	}
-
-	return toObjectErr(err, bucket)
-}
-
 // markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
 // syncs the delete to peers
 func (er erasureObjects) markDelete(ctx context.Context, bucket, prefix string) error {

cmd/erasure-healing.go

@@ -235,6 +235,10 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 				// we ignore disk not found errors
 				return nil
 			}
+			if storageDisks[index].Healing() != nil {
+				// we ignore disks under healing
+				return nil
+			}
 			volsInfo, err := storageDisks[index].ListVols(ctx)
 			if err != nil {
 				return err

cmd/erasure-object_test.go

@@ -214,22 +214,19 @@ func TestDeleteObjectsVersioned(t *testing.T) {
 func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	var objs []*erasureObjects
-	for i := 0; i < 32; i++ {
-		obj, fsDirs, err := prepareErasure(ctx, 16)
-		if err != nil {
-			t.Fatal("Unable to initialize 'Erasure' object layer.", err)
-		}
-		// Remove all dirs.
-		for _, dir := range fsDirs {
-			defer os.RemoveAll(dir)
-		}
-		z := obj.(*erasureServerPools)
-		xl := z.serverPools[0].sets[0]
-		objs = append(objs, xl)
+
+	obj, fsDirs, err := prepareErasureSets32(ctx)
+	if err != nil {
+		t.Fatal("Unable to initialize 'Erasure' object layer.", err)
 	}
 
-	erasureSets := &erasureSets{sets: objs, distributionAlgo: "CRCMOD"}
+	setObjectLayer(obj)
+	initConfigSubsystem(ctx, obj)
+
+	// Remove all dirs.
+	for _, dir := range fsDirs {
+		defer os.RemoveAll(dir)
+	}
 
 	type testCaseType struct {
 		bucket string
@@ -244,13 +241,12 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 		{bucketName, "obj_4"},
 	}
 
-	err := erasureSets.MakeBucket(ctx, bucketName, MakeBucketOptions{})
-	if err != nil {
+	if err = obj.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
 	for _, testCase := range testCases {
-		_, err = erasureSets.PutObject(ctx, testCase.bucket, testCase.object,
+		_, err = obj.PutObject(ctx, testCase.bucket, testCase.object,
 			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)
@@ -270,7 +266,7 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	}
 
 	objectNames := toObjectNames(testCases)
-	_, delErrs := erasureSets.DeleteObjects(ctx, bucketName, objectNames, ObjectOptions{})
+	_, delErrs := obj.DeleteObjects(ctx, bucketName, objectNames, ObjectOptions{})
 
 	for i := range delErrs {
 		if delErrs[i] != nil {
@@ -279,7 +275,7 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	}
 
 	for _, test := range testCases {
-		_, statErr := erasureSets.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{})
+		_, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{})
 		switch statErr.(type) {
 		case ObjectNotFound:
 		default:

cmd/erasure-server-pool-decom.go

@@ -686,6 +686,39 @@ func (v versionsSorter) reverse() {
 	})
 }
 
+func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi decomBucketInfo, fn func(entry metaCacheEntry)) error {
+	disks := set.getOnlineDisks()
+	if len(disks) == 0 {
+		return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints())
+	}
+
+	// How to resolve partial results.
+	resolver := metadataResolutionParams{
+		dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+		objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+		bucket:    bi.Name,
+	}
+
+	err := listPathRaw(ctx, listPathRawOptions{
+		disks:          disks,
+		bucket:         bi.Name,
+		path:           bi.Prefix,
+		recursive:      true,
+		forwardTo:      "",
+		minDisks:       len(disks) / 2, // to capture all quorum ratios
+		reportNotFound: false,
+		agreed:         fn,
+		partial: func(entries metaCacheEntries, _ []error) {
+			entry, ok := entries.resolve(&resolver)
+			if ok {
+				fn(*entry)
+			}
+		},
+		finished: nil,
+	})
+	return err
+}
+
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
 
@@ -700,12 +733,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 	for _, set := range pool.sets {
 		set := set
-		disks := set.getOnlineDisks()
-		if len(disks) == 0 {
-			logger.LogIf(GlobalContext, fmt.Errorf("no online drives found for set with endpoints %s",
-				set.getEndpoints()))
-			continue
-		}
 
 		vc, _ := globalBucketVersioningSys.Get(bi.Name)
 
@@ -879,39 +906,16 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			z.poolMetaMutex.Unlock()
 		}
 
-		// How to resolve partial results.
-		resolver := metadataResolutionParams{
-			dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
-			objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
-			bucket:    bi.Name,
-		}
-
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err := listPathRaw(ctx, listPathRawOptions{
-				disks:          disks,
-				bucket:         bi.Name,
-				path:           bi.Prefix,
-				recursive:      true,
-				forwardTo:      "",
-				minDisks:       len(disks) / 2, // to capture all quorum ratios
-				reportNotFound: false,
-				agreed: func(entry metaCacheEntry) {
+			err := set.listObjectsToDecommission(ctx, bi,
+				func(entry metaCacheEntry) {
 					parallelWorkers <- struct{}{}
 					wg.Add(1)
 					go decommissionEntry(entry)
 				},
-				partial: func(entries metaCacheEntries, _ []error) {
-					entry, ok := entries.resolve(&resolver)
-					if ok {
-						parallelWorkers <- struct{}{}
-						wg.Add(1)
-						go decommissionEntry(*entry)
-					}
-				},
-				finished: nil,
-			})
+			)
 			logger.LogIf(ctx, err)
 		}()
 	}
@@ -992,6 +996,33 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 	return nil
 }
 
+func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error {
+	buckets, err := z.getBucketsToDecommission(ctx)
+	if err != nil {
+		return err
+	}
+
+	pool := z.serverPools[idx]
+	for _, set := range pool.sets {
+		for _, bi := range buckets {
+			var objectsFound int
+			err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
+				if entry.isObject() {
+					objectsFound++
+				}
+			})
+			if err != nil {
+				return err
+			}
+			if objectsFound > 0 {
+				return fmt.Errorf("at least %d objects were found in bucket `%s` after decommissioning", objectsFound, bi.Name)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx int) {
 	z.poolMetaMutex.Lock()
 	var dctx context.Context
@@ -1011,6 +1042,15 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	failed := z.poolMeta.Pools[idx].Decommission.ItemsDecommissionFailed > 0
 	z.poolMetaMutex.Unlock()
 
+	if !failed {
+		logger.Info("Decommissioning almost complete - checking for left over objects")
+		err := z.checkAfterDecom(dctx, idx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			failed = true
+		}
+	}
+
 	if failed {
 		// Decommission failed indicate as such.
 		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
@@ -1193,6 +1233,33 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 	return nil
 }
 
+func (z *erasureServerPools) getBucketsToDecommission(ctx context.Context) ([]decomBucketInfo, error) {
+	buckets, err := z.ListBuckets(ctx, BucketOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	decomBuckets := make([]decomBucketInfo, len(buckets))
+	for i := range buckets {
+		decomBuckets[i] = decomBucketInfo{
+			Name: buckets[i].Name,
+		}
+	}
+
+	// Buckets data are dispersed in multiple zones/sets, make
+	// sure to decommission the necessary metadata.
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: minioConfigPrefix,
+	})
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: bucketMetaPrefix,
+	})
+
+	return decomBuckets, nil
+}
+
 func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (err error) {
 	if idx < 0 {
 		return errInvalidArgument
@@ -1202,25 +1269,15 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		return errInvalidArgument
 	}
 
-	buckets, err := z.ListBuckets(ctx, BucketOptions{})
+	decomBuckets, err := z.getBucketsToDecommission(ctx)
 	if err != nil {
 		return err
 	}
 
-	// Make sure to heal the buckets to ensure the new
-	// pool has the new buckets, this is to avoid
-	// failures later.
-	for _, bucket := range buckets {
+	for _, bucket := range decomBuckets {
 		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
 	}
 
-	decomBuckets := make([]decomBucketInfo, len(buckets))
-	for i := range buckets {
-		decomBuckets[i] = decomBucketInfo{
-			Name: buckets[i].Name,
-		}
-	}
-
 	// TODO: Support decommissioning transition tiers.
 	for _, bucket := range decomBuckets {
 		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
@@ -1247,17 +1304,6 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		}
 	}
 
-	// Buckets data are dispersed in multiple zones/sets, make
-	// sure to decommission the necessary metadata.
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: minioConfigPrefix,
-	})
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: bucketMetaPrefix,
-	})
-
 	var pool *erasureSets
 	for pidx := range z.serverPools {
 		if pidx == idx {

cmd/erasure-server-pool.go

@@ -61,6 +61,8 @@ type erasureServerPools struct {
 
 	// Active decommission canceler
 	decommissionCancelers []context.CancelFunc
+
+	s3Peer *S3PeerSys
 }
 
 func (z *erasureServerPools) SinglePool() bool {
@@ -79,6 +81,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		storageDisks = make([][]StorageAPI, len(endpointServerPools))
 		z            = &erasureServerPools{
 			serverPools: make([]*erasureSets, len(endpointServerPools)),
+			s3Peer:      NewS3PeerSys(endpointServerPools),
 		}
 	)
 
@@ -91,11 +94,14 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		// -- Default for Standard Storage class is, parity = 3 - disks 6, 7
 		// -- Default for Standard Storage class is, parity = 4 - disks 8 to 16
 		if commonParityDrives == 0 {
-			commonParityDrives = ecDrivesNoConfig(ep.DrivesPerSet)
+			commonParityDrives, err = ecDrivesNoConfig(ep.DrivesPerSet)
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		if err = storageclass.ValidateParity(commonParityDrives, ep.DrivesPerSet); err != nil {
-			return nil, fmt.Errorf("parity validation returned an error %w <- (%d, %d), for pool(%s)", err, commonParityDrives, ep.DrivesPerSet, humanize.Ordinal(i+1))
+			return nil, fmt.Errorf("parity validation returned an error: %w <- (%d, %d), for pool(%s)", err, commonParityDrives, ep.DrivesPerSet, humanize.Ordinal(i+1))
 		}
 
 		storageDisks[i], formats[i], err = waitForFormatErasure(local, ep.Endpoints, i+1,
@@ -710,49 +716,34 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
-	g := errgroup.WithNErrs(len(z.serverPools))
-
+	// Verify if bucket is valid.
 	if !isMinioMetaBucketName(bucket) {
-		// Verify if bucket is valid.
 		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
 			return BucketNameInvalid{Bucket: bucket}
 		}
 
-		// Lock the bucket name before creating.
-		lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
-		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-		if err != nil {
-			return err
-		}
+		if !opts.NoLock {
+			// Lock the bucket name before creating.
+			lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
+			lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
+				return err
+			}
 
-		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx)
+			ctx = lkctx.Context()
+			defer lk.Unlock(lkctx)
+		}
 	}
 
-	// Create buckets in parallel across all sets.
-	for index := range z.serverPools {
-		index := index
-		g.Go(func() error {
-			if z.IsSuspended(index) {
-				return nil
-			}
-			return z.serverPools[index].MakeBucket(ctx, bucket, opts)
-		}, index)
-	}
-
-	errs := g.Wait()
-	// Return the first encountered error
-	for _, err := range errs {
-		if err != nil {
-			if _, ok := err.(BucketExists); !ok {
-				// Delete created buckets, ignoring errors.
-				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
-					Force:      false,
-					NoRecreate: true,
-				})
-			}
-			return err
+	if err := z.s3Peer.MakeBucket(ctx, bucket, opts); err != nil {
+		if _, ok := err.(BucketExists); !ok {
+			// Delete created buckets, ignoring errors.
+			z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+				NoLock:     true,
+				NoRecreate: true,
+			})
 		}
+		return err
 	}
 
 	// If it doesn't exist we get a new, so ignore errors
@@ -1609,38 +1600,18 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 
 // GetBucketInfo - returns bucket info from one of the erasure coded serverPools.
 func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
-	if z.SinglePool() {
-		bucketInfo, err = z.serverPools[0].GetBucketInfo(ctx, bucket, opts)
-		if err != nil {
-			return bucketInfo, err
-		}
-		meta, err := globalBucketMetadataSys.Get(bucket)
-		if err == nil {
-			bucketInfo.Created = meta.Created
-			bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
-			bucketInfo.ObjectLocking = meta.LockEnabled
-		}
-		return bucketInfo, nil
+	bucketInfo, err = z.s3Peer.GetBucketInfo(ctx, bucket, opts)
+	if err != nil {
+		return bucketInfo, toObjectErr(err, bucket)
 	}
-	for _, pool := range z.serverPools {
-		bucketInfo, err = pool.GetBucketInfo(ctx, bucket, opts)
-		if err != nil {
-			if isErrBucketNotFound(err) {
-				continue
-			}
-			return bucketInfo, err
-		}
-		meta, err := globalBucketMetadataSys.Get(bucket)
-		if err == nil {
-			bucketInfo.Created = meta.Created
-			bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
-			bucketInfo.ObjectLocking = meta.LockEnabled
-		}
-		return bucketInfo, nil
-	}
-	return bucketInfo, BucketNotFound{
-		Bucket: bucket,
+
+	meta, err := globalBucketMetadataSys.Get(bucket)
+	if err == nil {
+		bucketInfo.Created = meta.Created
+		bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
+		bucketInfo.ObjectLocking = meta.LockEnabled
 	}
+	return bucketInfo, nil
 }
 
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
@@ -1671,43 +1642,48 @@ func (z *erasureServerPools) IsTaggingSupported() bool {
 // even if one of the serverPools fail to delete buckets, we proceed to
 // undo a successful operation.
 func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
-	defer NSUpdated(bucket, slashSeparator)
-
-	g := errgroup.WithNErrs(len(z.serverPools))
-
-	// Delete buckets in parallel across all serverPools.
-	for index := range z.serverPools {
-		index := index
-		g.Go(func() error {
-			if z.IsSuspended(index) {
-				return nil
-			}
-			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
-		}, index)
+	if isMinioMetaBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
 	}
 
-	errs := g.Wait()
+	// Verify if bucket is valid.
+	if err := s3utils.CheckValidBucketName(bucket); err != nil {
+		return BucketNameInvalid{Bucket: bucket}
+	}
 
-	// For any write quorum failure, we undo all the delete
-	// buckets operation by creating all the buckets again.
-	for _, err := range errs {
+	defer NSUpdated(bucket, slashSeparator)
+	if !opts.NoLock {
+		// Lock the bucket name before creating.
+		lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
-			if !z.SinglePool() && !opts.NoRecreate {
-				undoDeleteBucketServerPools(context.Background(), bucket, z.serverPools, errs)
-			}
 			return err
 		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
+	err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
+	if err == nil || errors.Is(err, errVolumeNotFound) {
+		// If site replication is configured, hold on to deleted bucket state until sites sync
+		switch opts.SRDeleteOp {
+		case MarkDelete:
+			z.markDelete(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+		}
 	}
 
-	// Purge the entire bucket metadata entirely.
-	z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
-	// If site replication is configured, hold on to deleted bucket state until sites sync
-	switch opts.SRDeleteOp {
-	case MarkDelete:
-		z.markDelete(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+	if err != nil && !errors.Is(err, errVolumeNotFound) {
+		if !opts.NoRecreate {
+			z.s3Peer.MakeBucket(ctx, bucket, MakeBucketOptions{})
+		}
 	}
-	// Success.
-	return nil
+
+	if err == nil {
+		// Purge the entire bucket metadata entirely.
+		z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
+	}
+
+	return toObjectErr(err, bucket)
 }
 
 // deleteAll will rename bucket+prefix unconditionally across all disks to
@@ -1744,43 +1720,11 @@ func (z *erasureServerPools) purgeDelete(ctx context.Context, bucket, prefix str
 	}
 }
 
-// This function is used to undo a successful DeleteBucket operation.
-func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools []*erasureSets, errs []error) {
-	g := errgroup.WithNErrs(len(serverPools))
-
-	// Undo previous delete bucket on all underlying serverPools.
-	for index := range serverPools {
-		index := index
-		g.Go(func() error {
-			if errs[index] == nil {
-				return serverPools[index].MakeBucket(ctx, bucket, MakeBucketOptions{})
-			}
-			return nil
-		}, index)
-	}
-
-	g.Wait()
-}
-
 // List all buckets from one of the serverPools, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all serverPools.
 func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
-	if z.SinglePool() {
-		buckets, err = z.serverPools[0].ListBuckets(ctx, opts)
-	} else {
-		for idx, pool := range z.serverPools {
-			if z.IsSuspended(idx) {
-				continue
-			}
-			buckets, err = pool.ListBuckets(ctx, opts)
-			if err != nil {
-				logger.LogIf(ctx, err)
-				continue
-			}
-			break
-		}
-	}
+	buckets, err = z.s3Peer.ListBuckets(ctx, opts)
 	if err != nil {
 		return nil, err
 	}

cmd/erasure-sets.go

@@ -26,7 +26,6 @@ import (
 	"math/rand"
 	"net/http"
 	"reflect"
-	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -355,10 +354,6 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 		endpointStrings[i] = endpoint.String()
 	}
 
-	if defaultParityCount == 0 {
-		logger.Error("Warning: Default parity set to 0. This can lead to data loss.")
-	}
-
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
 		sets:               make([]*erasureObjects, setCount),
@@ -677,32 +672,6 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-// MakeBucketLocation - creates a new bucket across all sets simultaneously,
-// then return the first encountered error
-func (s *erasureSets) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
-	g := errgroup.WithNErrs(len(s.sets))
-
-	// Create buckets in parallel across all sets.
-	for index := range s.sets {
-		index := index
-		g.Go(func() error {
-			return s.sets[index].MakeBucket(ctx, bucket, opts)
-		}, index)
-	}
-
-	errs := g.Wait()
-
-	// Return the first encountered error
-	for _, err := range errs {
-		if err != nil {
-			return err
-		}
-	}
-
-	// Success.
-	return nil
-}
-
 // hashes the key returning an integer based on the input algorithm.
 // This function currently supports
 // - CRCMOD
@@ -749,11 +718,6 @@ func (s *erasureSets) getHashedSet(input string) (set *erasureObjects) {
 	return s.sets[s.getHashedSetIndex(input)]
 }
 
-// GetBucketInfo - returns bucket info from one of the erasure coded set.
-func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
-	return s.getHashedSet("").GetBucketInfo(ctx, bucket, opts)
-}
-
 // IsNotificationSupported returns whether bucket notification is applicable for this layer.
 func (s *erasureSets) IsNotificationSupported() bool {
 	return s.getHashedSet("").IsNotificationSupported()
@@ -778,102 +742,6 @@ func (s *erasureSets) IsTaggingSupported() bool {
 	return true
 }
 
-// DeleteBucket - deletes a bucket on all sets simultaneously,
-// even if one of the sets fail to delete buckets, we proceed to
-// undo a successful operation.
-func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
-	g := errgroup.WithNErrs(len(s.sets))
-
-	// Delete buckets in parallel across all sets.
-	for index := range s.sets {
-		index := index
-		g.Go(func() error {
-			return s.sets[index].DeleteBucket(ctx, bucket, opts)
-		}, index)
-	}
-
-	errs := g.Wait()
-	// For any failure, we attempt undo all the delete buckets operation
-	// by creating buckets again on all sets which were successfully deleted.
-	for _, err := range errs {
-		if err != nil && !opts.NoRecreate {
-			undoDeleteBucketSets(ctx, bucket, s.sets, errs)
-			return err
-		}
-	}
-
-	// Success.
-	return nil
-}
-
-// This function is used to undo a successful DeleteBucket operation.
-func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObjects, errs []error) {
-	g := errgroup.WithNErrs(len(sets))
-
-	// Undo previous delete bucket on all underlying sets.
-	for index := range sets {
-		index := index
-		g.Go(func() error {
-			if errs[index] == nil {
-				return sets[index].MakeBucket(ctx, bucket, MakeBucketOptions{})
-			}
-			return nil
-		}, index)
-	}
-
-	g.Wait()
-}
-
-// List all buckets from one of the set, we are not doing merge
-// sort here just for simplification. As per design it is assumed
-// that all buckets are present on all sets.
-func (s *erasureSets) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
-	var listBuckets []BucketInfo
-	healBuckets := map[string]VolInfo{}
-	for _, set := range s.sets {
-		// lists all unique buckets across drives.
-		if err := listAllBuckets(ctx, set.getDisks(), healBuckets, s.defaultParityCount); err != nil {
-			return nil, err
-		}
-	}
-
-	// include deleted buckets in listBuckets output
-	deletedBuckets := map[string]VolInfo{}
-
-	if opts.Deleted {
-		for _, set := range s.sets {
-			// lists all deleted buckets across drives.
-			if err := listDeletedBuckets(ctx, set.getDisks(), deletedBuckets, s.defaultParityCount); err != nil {
-				return nil, err
-			}
-		}
-	}
-	for _, v := range healBuckets {
-		bi := BucketInfo{
-			Name:    v.Name,
-			Created: v.Created,
-		}
-		if vi, ok := deletedBuckets[v.Name]; ok {
-			bi.Deleted = vi.Created
-		}
-		listBuckets = append(listBuckets, bi)
-	}
-	for _, v := range deletedBuckets {
-		if _, ok := healBuckets[v.Name]; !ok {
-			listBuckets = append(listBuckets, BucketInfo{
-				Name:    v.Name,
-				Deleted: v.Created,
-			})
-		}
-	}
-
-	sort.Slice(listBuckets, func(i, j int) bool {
-		return listBuckets[i].Name < listBuckets[j].Name
-	})
-
-	return listBuckets, nil
-}
-
 // listDeletedBuckets lists deleted buckets from all disks.
 func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBuckets map[string]VolInfo, readQuorum int) error {
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -1277,6 +1145,8 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		return res, errNoHealRequired
 	}
 
+	formatOpID := mustGetUUID()
+
 	// Initialize a new set of set formats which will be written to disk.
 	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs)
 
@@ -1298,7 +1168,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			if storageDisks[index] == nil || format == nil {
 				continue
 			}
-			if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+			if err := saveFormatErasure(storageDisks[index], format, formatOpID); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
 				tmpNewFormats[index] = nil // this disk failed to write new format
 			}

cmd/erasure-sets_test.go

@@ -191,7 +191,12 @@ func TestNewErasureSets(t *testing.T) {
 	}
 
 	ep := PoolEndpoints{Endpoints: endpoints}
-	if _, err := newErasureSets(ctx, ep, storageDisks, format, ecDrivesNoConfig(16), 0); err != nil {
+
+	parity, err := ecDrivesNoConfig(16)
+	if err != nil {
+		t.Fatalf("Unexpected error during EC drive config: %v", err)
+	}
+	if _, err := newErasureSets(ctx, ep, storageDisks, format, parity, 0); err != nil {
 		t.Fatalf("Unable to initialize erasure")
 	}
 }

cmd/erasure.go

@@ -89,10 +89,6 @@ func (er erasureObjects) defaultWQuorum() int {
 	return dataCount
 }
 
-func (er erasureObjects) defaultRQuorum() int {
-	return er.setDriveCount - er.defaultParityCount
-}
-
 // byDiskTotal is a collection satisfying sort.Interface.
 type byDiskTotal []madmin.Disk
 

cmd/format-erasure.go

@@ -347,7 +347,7 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 	return formats, g.Wait()
 }
 
-func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) error {
+func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string) error {
 	if disk == nil || format == nil {
 		return errDiskNotFound
 	}
@@ -383,9 +383,9 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	}
 
 	disk.SetDiskID(diskID)
-	if heal {
+	if healID != "" {
 		ctx := context.Background()
-		ht := newHealingTracker(disk)
+		ht := newHealingTracker(disk, healID)
 		return ht.save(ctx)
 	}
 	return nil
@@ -541,7 +541,7 @@ func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []Stora
 				}
 				format.ID = refFormat.ID
 				// Heal the drive if we fixed its deployment ID.
-				if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+				if err := saveFormatErasure(storageDisks[index], format, mustGetUUID()); err != nil {
 					logger.LogIf(GlobalContext, err)
 					return fmt.Errorf("Unable to save format.json, %w", err)
 				}
@@ -642,7 +642,7 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 			if formats[index] == nil {
 				return errDiskNotFound
 			}
-			return saveFormatErasure(storageDisks[index], formats[index], false)
+			return saveFormatErasure(storageDisks[index], formats[index], "")
 		}, index)
 	}
 
@@ -722,7 +722,7 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 			if formats[i].Erasure.This == "" {
 				formats[i].Erasure.This = formats[i].Erasure.Sets[0][i]
 				// Heal the drive if drive has .This empty.
-				if err := saveFormatErasure(storageDisks[i], formats[i], true); err != nil {
+				if err := saveFormatErasure(storageDisks[i], formats[i], mustGetUUID()); err != nil {
 					return err
 				}
 			}
@@ -741,7 +741,10 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID, distributionAlgo string, sErrs []error) (*formatErasureV3, error) {
 	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, len(storageDisks))
-	wantAtMost := ecDrivesNoConfig(setDriveCount)
+	wantAtMost, err := ecDrivesNoConfig(setDriveCount)
+	if err != nil {
+		return nil, err
+	}
 
 	for i := 0; i < setCount; i++ {
 		hostCount := make(map[string]int, setDriveCount)
@@ -795,13 +798,12 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 
 // ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
 // It will attempt to read it from env variable and fall back to drives/2.
-func ecDrivesNoConfig(setDriveCount int) int {
-	sc, _ := storageclass.LookupConfig(config.KVS{}, setDriveCount)
-	ecDrives := sc.GetParityForSC(storageclass.STANDARD)
-	if ecDrives < 0 {
-		ecDrives = storageclass.DefaultParityBlocks(setDriveCount)
+func ecDrivesNoConfig(setDriveCount int) (int, error) {
+	sc, err := storageclass.LookupConfig(config.KVS{}, setDriveCount)
+	if err != nil {
+		return 0, err
 	}
-	return ecDrives
+	return sc.GetParityForSC(storageclass.STANDARD), nil
 }
 
 // Make Erasure backend meta volumes.

cmd/iam-store.go

@@ -2133,6 +2133,10 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		cr.SecretKey = opts.secretKey
 	}
 
+	if opts.comment != "" {
+		cr.Comment = opts.comment
+	}
+
 	switch opts.status {
 	// The caller did not ask to update status account, do nothing
 	case "":

cmd/iam.go

@@ -919,6 +919,7 @@ type newServiceAccountOpts struct {
 	sessionPolicy *iampolicy.Policy
 	accessKey     string
 	secretKey     string
+	comment       string
 
 	claims map[string]interface{}
 }
@@ -989,6 +990,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	cred.ParentUser = parentUser
 	cred.Groups = groups
 	cred.Status = string(auth.AccountOn)
+	cred.Comment = opts.comment
 
 	updatedAt, err := sys.store.AddServiceAccount(ctx, cred)
 	if err != nil {
@@ -1003,6 +1005,7 @@ type updateServiceAccountOpts struct {
 	sessionPolicy *iampolicy.Policy
 	secretKey     string
 	status        string
+	comment       string
 }
 
 // UpdateServiceAccount - edit a service account

cmd/object-api-interface.go

@@ -121,12 +121,14 @@ type MakeBucketOptions struct {
 	VersioningEnabled bool
 	ForceCreate       bool      // Create buckets even if they are already created.
 	CreatedAt         time.Time // only for site replication
+	NoLock            bool      // does not lock the make bucket call if set to 'true'
 }
 
 // DeleteBucketOptions provides options for DeleteBucket calls.
 type DeleteBucketOptions struct {
+	NoLock     bool             // does not lock the delete bucket call if set to 'true'
+	NoRecreate bool             // do not recreate bucket on delete failures
 	Force      bool             // Force deletion
-	NoRecreate bool             // Do not recreate on delete failures
 	SRDeleteOp SRBucketDeleteOp // only when site replication is enabled
 }
 

cmd/object-api-utils.go

@@ -170,6 +170,13 @@ func IsValidObjectPrefix(object string) bool {
 	if strings.Contains(object, `//`) {
 		return false
 	}
+	// This is valid for AWS S3 but it will never
+	// work with file systems, we will reject here
+	// to return object name invalid rather than
+	// a cryptic error from the file system.
+	if strings.ContainsRune(object, 0) {
+		return false
+	}
 	return true
 }
 

cmd/peer-s3-client.go

@@ -0,0 +1,346 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"io"
+	"net/url"
+	"strconv"
+
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/rest"
+	"github.com/minio/minio/internal/sync/errgroup"
+	xnet "github.com/minio/pkg/net"
+)
+
+var errPeerOffline = errors.New("peer is offline")
+
+// client to talk to peer Nodes.
+type peerS3Client struct {
+	host       *xnet.Host
+	restClient *rest.Client
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *peerS3Client) call(method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	return client.callWithContext(GlobalContext, method, values, body, length)
+}
+
+// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
+// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
+// after verifying format.json
+func (client *peerS3Client) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
+	if values == nil {
+		values = make(url.Values)
+	}
+
+	respBody, err = client.restClient.Call(ctx, method, values, body, length)
+	if err == nil {
+		return respBody, nil
+	}
+
+	err = toStorageErr(err)
+	return nil, err
+}
+
+// S3PeerSys - S3 peer call system.
+type S3PeerSys struct {
+	peerClients    []*peerS3Client // Excludes self
+	allPeerClients []*peerS3Client // Includes nil client for self
+}
+
+// NewS3PeerSys - creates new S3 peer calls.
+func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys {
+	remote, all := newPeerS3Clients(endpoints)
+	return &S3PeerSys{
+		peerClients:    remote,
+		allPeerClients: all,
+	}
+}
+
+// ListBuckets lists buckets across all servers and returns a possible consistent view
+func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) (result []BucketInfo, err error) {
+	g := errgroup.WithNErrs(len(sys.peerClients))
+
+	localBuckets, err := listBucketsLocal(ctx, opts)
+	if err != nil {
+		return nil, err
+	}
+
+	nodeBuckets := make([][]BucketInfo, len(sys.peerClients)+1)
+	errs := []error{nil}
+	nodeBuckets[0] = localBuckets
+
+	for idx, client := range sys.peerClients {
+		idx := idx
+		client := client
+		g.Go(func() error {
+			if client == nil {
+				return errPeerOffline
+			}
+			localBuckets, err := client.ListBuckets(ctx, opts)
+			if err != nil {
+				return err
+			}
+			nodeBuckets[idx+1] = localBuckets
+			return nil
+		}, idx)
+	}
+
+	errs = append(errs, g.Wait()...)
+
+	quorum := (len(sys.allPeerClients) / 2)
+	if err = reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum); err != nil {
+		return nil, err
+	}
+
+	bucketsMap := make(map[string]struct{})
+	for idx, buckets := range nodeBuckets {
+		if errs[idx] != nil {
+			continue
+		}
+		for _, bi := range buckets {
+			_, ok := bucketsMap[bi.Name]
+			if !ok {
+				bucketsMap[bi.Name] = struct{}{}
+				result = append(result, bi)
+			}
+		}
+	}
+
+	return result, nil
+}
+
+// GetBucketInfo returns bucket stat info about bucket on disk across all peers
+func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (binfo BucketInfo, err error) {
+	g := errgroup.WithNErrs(len(sys.peerClients))
+
+	bucketInfos := make([]BucketInfo, len(sys.peerClients)+1)
+
+	bucketInfo, err := getBucketInfoLocal(ctx, bucket, opts)
+	if err != nil {
+		return BucketInfo{}, err
+	}
+
+	errs := []error{nil}
+	bucketInfos[0] = bucketInfo
+
+	for idx, client := range sys.peerClients {
+		idx := idx
+		client := client
+		g.Go(func() error {
+			if client == nil {
+				return errPeerOffline
+			}
+			bucketInfo, err := client.GetBucketInfo(ctx, bucket, opts)
+			if err != nil {
+				return err
+			}
+			bucketInfos[idx] = bucketInfo
+			return nil
+		}, idx)
+	}
+
+	errs = append(errs, g.Wait()...)
+
+	quorum := (len(sys.allPeerClients) / 2)
+	if err = reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum); err != nil {
+		return BucketInfo{}, err
+	}
+
+	for i, err := range errs {
+		if err == nil {
+			bucketInfo = bucketInfos[i]
+			break
+		}
+	}
+
+	return bucketInfo, nil
+}
+
+func (client *peerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
+	v := url.Values{}
+	v.Set(peerS3BucketDeleted, strconv.FormatBool(opts.Deleted))
+
+	respBody, err := client.call(peerS3MethodListBuckets, v, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+	defer xhttp.DrainBody(respBody)
+
+	var buckets []BucketInfo
+	err = gob.NewDecoder(respBody).Decode(&buckets)
+	return buckets, err
+}
+
+// GetBucketInfo returns bucket stat info from a peer
+func (client *peerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
+	v := url.Values{}
+	v.Set(peerS3Bucket, bucket)
+	v.Set(peerS3BucketDeleted, strconv.FormatBool(opts.Deleted))
+
+	respBody, err := client.call(peerS3MethodGetBucketInfo, v, nil, -1)
+	if err != nil {
+		return BucketInfo{}, err
+	}
+	defer xhttp.DrainBody(respBody)
+
+	var bucketInfo BucketInfo
+	err = gob.NewDecoder(respBody).Decode(&bucketInfo)
+	return bucketInfo, err
+}
+
+// MakeBucket creates bucket across all peers
+func (sys *S3PeerSys) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
+	g := errgroup.WithNErrs(len(sys.peerClients))
+
+	for idx, client := range sys.peerClients {
+		client := client
+		g.Go(func() error {
+			if client == nil {
+				return errPeerOffline
+			}
+			return client.MakeBucket(ctx, bucket, opts)
+		}, idx)
+	}
+
+	errs := g.Wait()
+	errs = append(errs, makeBucketLocal(ctx, bucket, opts))
+
+	quorum := (len(sys.allPeerClients) / 2) + 1
+	err := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum)
+	return toObjectErr(err, bucket)
+}
+
+// MakeBucket creates a bucket on a peer
+func (client *peerS3Client) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
+	v := url.Values{}
+	v.Set(peerS3Bucket, bucket)
+	v.Set(peerS3BucketForceCreate, strconv.FormatBool(opts.ForceCreate))
+
+	respBody, err := client.call(peerS3MethodMakeBucket, v, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer xhttp.DrainBody(respBody)
+
+	return nil
+}
+
+// DeleteBucket deletes bucket across all peers
+func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
+	g := errgroup.WithNErrs(len(sys.peerClients))
+
+	for idx, client := range sys.peerClients {
+		client := client
+		g.Go(func() error {
+			if client == nil {
+				return errPeerOffline
+			}
+			return client.DeleteBucket(ctx, bucket, opts)
+		}, idx)
+	}
+
+	errs := g.Wait()
+	errs = append(errs, deleteBucketLocal(ctx, bucket, opts))
+
+	for _, err := range errs {
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// DeleteBucket deletes bucket on a peer
+func (client *peerS3Client) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
+	v := url.Values{}
+	v.Set(peerS3Bucket, bucket)
+	v.Set(peerS3BucketForceDelete, strconv.FormatBool(opts.Force))
+
+	respBody, err := client.call(peerS3MethodDeleteBucket, v, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer xhttp.DrainBody(respBody)
+
+	return nil
+}
+
+// newPeerS3Clients creates new peer clients.
+// The two slices will point to the same clients,
+// but 'all' will contain nil entry for local client.
+// The 'all' slice will be in the same order across the cluster.
+func newPeerS3Clients(endpoints EndpointServerPools) (remote, all []*peerS3Client) {
+	if !globalIsDistErasure {
+		// Only useful in distributed setups
+		return nil, nil
+	}
+	hosts := endpoints.hostsSorted()
+	remote = make([]*peerS3Client, 0, len(hosts))
+	all = make([]*peerS3Client, len(hosts))
+	for i, host := range hosts {
+		if host == nil {
+			continue
+		}
+		all[i] = newPeerS3Client(host)
+		remote = append(remote, all[i])
+	}
+	if len(all) != len(remote)+1 {
+		logger.LogIf(context.Background(), fmt.Errorf("WARNING: Expected number of all hosts (%v) to be remote +1 (%v)", len(all), len(remote)))
+	}
+	return remote, all
+}
+
+// Returns a peer S3 client.
+func newPeerS3Client(peer *xnet.Host) *peerS3Client {
+	scheme := "http"
+	if globalIsTLS {
+		scheme = "https"
+	}
+
+	serverURL := &url.URL{
+		Scheme: scheme,
+		Host:   peer.String(),
+		Path:   peerS3Path,
+	}
+
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
+	// Use a separate client to avoid recursive calls.
+	healthClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
+	healthClient.ExpectTimeouts = true
+	healthClient.NoMetrics = true
+
+	// Construct a new health function.
+	restClient.HealthCheckFn = func() bool {
+		ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout)
+		defer cancel()
+		respBody, err := healthClient.Call(ctx, peerS3MethodHealth, nil, nil, -1)
+		xhttp.DrainBody(respBody)
+		return !isNetworkError(err)
+	}
+
+	return &peerS3Client{host: peer, restClient: restClient}
+}

cmd/peer-s3-server.go

@@ -0,0 +1,325 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/gob"
+	"errors"
+	"net/http"
+	"sort"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/sync/errgroup"
+)
+
+const (
+	peerS3Version = "v1" // First implementation
+
+	peerS3VersionPrefix = SlashSeparator + peerS3Version
+	peerS3Prefix        = minioReservedBucketPath + "/peer"
+	peerS3Path          = peerS3Prefix + peerS3VersionPrefix
+)
+
+const (
+	peerS3MethodHealth        = "/health"
+	peerS3MethodMakeBucket    = "/make-bucket"
+	peerS3MethodGetBucketInfo = "/get-bucket-info"
+	peerS3MethodDeleteBucket  = "/delete-bucket"
+	peerS3MethodListBuckets   = "/list-buckets"
+)
+
+const (
+	peerS3Bucket            = "bucket"
+	peerS3BucketDeleted     = "bucket-deleted"
+	peerS3BucketForceCreate = "force-create"
+	peerS3BucketForceDelete = "force-delete"
+)
+
+type peerS3Server struct{}
+
+func (s *peerS3Server) writeErrorResponse(w http.ResponseWriter, err error) {
+	w.WriteHeader(http.StatusForbidden)
+	w.Write([]byte(err.Error()))
+}
+
+// IsValid - To authenticate and verify the time difference.
+func (s *peerS3Server) IsValid(w http.ResponseWriter, r *http.Request) bool {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return false
+	}
+
+	if err := storageServerRequestValidate(r); err != nil {
+		s.writeErrorResponse(w, err)
+		return false
+	}
+	return true
+}
+
+// HealthHandler - returns true of health
+func (s *peerS3Server) HealthHandler(w http.ResponseWriter, r *http.Request) {
+	s.IsValid(w, r)
+}
+
+func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
+	quorum := (len(globalLocalDrives) / 2)
+
+	buckets = make([]BucketInfo, 0, 32)
+	healBuckets := map[string]VolInfo{}
+
+	// lists all unique buckets across drives.
+	if err := listAllBuckets(ctx, globalLocalDrives, healBuckets, quorum); err != nil {
+		return nil, err
+	}
+
+	// include deleted buckets in listBuckets output
+	deletedBuckets := map[string]VolInfo{}
+
+	if opts.Deleted {
+		// lists all deleted buckets across drives.
+		if err := listDeletedBuckets(ctx, globalLocalDrives, deletedBuckets, quorum); err != nil {
+			return nil, err
+		}
+	}
+
+	for _, v := range healBuckets {
+		bi := BucketInfo{
+			Name:    v.Name,
+			Created: v.Created,
+		}
+		if vi, ok := deletedBuckets[v.Name]; ok {
+			bi.Deleted = vi.Created
+		}
+		buckets = append(buckets, bi)
+	}
+
+	for _, v := range deletedBuckets {
+		if _, ok := healBuckets[v.Name]; !ok {
+			buckets = append(buckets, BucketInfo{
+				Name:    v.Name,
+				Deleted: v.Created,
+			})
+		}
+	}
+
+	sort.Slice(buckets, func(i, j int) bool {
+		return buckets[i].Name < buckets[j].Name
+	})
+
+	return buckets, nil
+}
+
+func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) {
+	g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
+	bucketsInfo := make([]BucketInfo, len(globalLocalDrives))
+
+	// Make a volume entry on all underlying storage disks.
+	for index := range globalLocalDrives {
+		index := index
+		g.Go(func() error {
+			if globalLocalDrives[index] == nil {
+				return errDiskNotFound
+			}
+			volInfo, err := globalLocalDrives[index].StatVol(ctx, bucket)
+			if err != nil {
+				if opts.Deleted {
+					dvi, derr := globalLocalDrives[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket))
+					if derr != nil {
+						return err
+					}
+					bucketsInfo[index] = BucketInfo{Name: bucket, Deleted: dvi.Created}
+					return nil
+				}
+				return err
+			}
+
+			bucketsInfo[index] = BucketInfo{Name: bucket, Created: volInfo.Created}
+			return nil
+		}, index)
+	}
+
+	errs := g.Wait()
+	if err := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(globalLocalDrives) / 2)); err != nil {
+		return BucketInfo{}, err
+	}
+
+	var bucketInfo BucketInfo
+	for i, err := range errs {
+		if err == nil {
+			bucketInfo = bucketsInfo[i]
+			break
+		}
+	}
+
+	return bucketInfo, nil
+}
+
+func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
+	g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
+
+	// Make a volume entry on all underlying storage disks.
+	for index := range globalLocalDrives {
+		index := index
+		g.Go(func() error {
+			if globalLocalDrives[index] == nil {
+				return errDiskNotFound
+			}
+			return globalLocalDrives[index].DeleteVol(ctx, bucket, opts.Force)
+		}, index)
+	}
+
+	var recreate bool
+	errs := g.Wait()
+	for index, err := range errs {
+		if errors.Is(err, errVolumeNotEmpty) {
+			recreate = true
+		}
+		if err == nil && recreate {
+			// ignore any errors
+			globalLocalDrives[index].MakeVol(ctx, bucket)
+		}
+	}
+
+	for _, err := range errs {
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func makeBucketLocal(ctx context.Context, bucket string, opts MakeBucketOptions) error {
+	g := errgroup.WithNErrs(len(globalLocalDrives)).WithConcurrency(32)
+
+	// Make a volume entry on all underlying storage disks.
+	for index := range globalLocalDrives {
+		index := index
+		g.Go(func() error {
+			if globalLocalDrives[index] == nil {
+				return errDiskNotFound
+			}
+			err := globalLocalDrives[index].MakeVol(ctx, bucket)
+			if opts.ForceCreate && errors.Is(err, errVolumeExists) {
+				// No need to return error when force create was
+				// requested.
+				return nil
+			}
+			if err != nil && !errors.Is(err, errVolumeExists) {
+				logger.LogIf(ctx, err)
+			}
+			return err
+		}, index)
+	}
+
+	errs := g.Wait()
+	return reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(globalLocalDrives)/2)+1)
+}
+
+func (s *peerS3Server) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+
+	bucketDeleted := r.Form.Get(peerS3BucketDeleted) == "true"
+
+	buckets, err := listBucketsLocal(r.Context(), BucketOptions{
+		Deleted: bucketDeleted,
+	})
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(buckets))
+}
+
+// GetBucketInfoHandler implements peer BuckeInfo call, returns bucket create date.
+func (s *peerS3Server) GetBucketInfoHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+
+	bucket := r.Form.Get(peerS3Bucket)
+	bucketDeleted := r.Form.Get(peerS3BucketDeleted) == "true"
+	bucketInfo, err := getBucketInfoLocal(r.Context(), bucket, BucketOptions{
+		Deleted: bucketDeleted,
+	})
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+
+	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(bucketInfo))
+}
+
+// DeleteBucketHandler implements peer delete bucket call.
+func (s *peerS3Server) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+
+	bucket := r.Form.Get(peerS3Bucket)
+	if isMinioMetaBucket(bucket) {
+		s.writeErrorResponse(w, errInvalidArgument)
+		return
+	}
+
+	forceDelete := r.Form.Get(peerS3BucketForceDelete) == "true"
+
+	err := deleteBucketLocal(r.Context(), bucket, DeleteBucketOptions{
+		Force: forceDelete,
+	})
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+}
+
+// MakeBucketHandler implements peer create bucket call.
+func (s *peerS3Server) MakeBucketHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+
+	bucket := r.Form.Get(peerS3Bucket)
+	forceCreate := r.Form.Get(peerS3BucketForceCreate) == "true"
+
+	err := makeBucketLocal(r.Context(), bucket, MakeBucketOptions{
+		ForceCreate: forceCreate,
+	})
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+}
+
+// registerPeerS3Handlers - register peer s3 router.
+func registerPeerS3Handlers(router *mux.Router) {
+	server := &peerS3Server{}
+	subrouter := router.PathPrefix(peerS3Prefix).Subrouter()
+
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodMakeBucket).HandlerFunc(httpTraceHdrs(server.MakeBucketHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodDeleteBucket).HandlerFunc(httpTraceHdrs(server.DeleteBucketHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodGetBucketInfo).HandlerFunc(httpTraceHdrs(server.GetBucketInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodListBuckets).HandlerFunc(httpTraceHdrs(server.ListBucketsHandler))
+}

cmd/routers.go

@@ -31,6 +31,9 @@ func registerDistErasureRouters(router *mux.Router, endpointServerPools Endpoint
 	// Register peer REST router only if its a distributed setup.
 	registerPeerRESTHandlers(router)
 
+	// Register peer S3 router only if its a distributed setup.
+	registerPeerS3Handlers(router)
+
 	// Register bootstrap REST router for distributed setups.
 	registerBootstrapRESTHandlers(router)
 

cmd/server-main.go

@@ -399,10 +399,14 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 		lkctx, err := txnLk.GetLock(ctx, lockTimeout)
 		if err != nil {
 			logger.Info("Waiting for all MinIO sub-systems to be initialized.. trying to acquire lock")
-			waitDuration := time.Duration(r.Float64() * 2 * float64(time.Second))
+			waitDuration := time.Duration(r.Float64() * 5 * float64(time.Second))
 			bootstrapTrace(fmt.Sprintf("lock not available. error: %v. sleeping for %v before retry", err, waitDuration))
 
-			// Sleep 0 -> 2 seconds to average 1 second retry interval.
+			// Sleep 0 -> 5 seconds, provider a higher range such that sleeps()
+			// and retries for lock are more spread out, needed orchestrated
+			// systems take 30s minimum to respond to DNS resolvers.
+			//
+			// Do not change this value.
 			time.Sleep(waitDuration)
 			continue
 		}
@@ -718,6 +722,11 @@ func serverMain(ctx *cli.Context) {
 
 		// Prints the formatted startup message, if err is not nil then it prints additional information as well.
 		printStartupMessage(getAPIEndpoints(), err)
+
+		// Print a warning at the end of the startup banner so it is more noticeable
+		if globalStorageClass.GetParityForSC("") == 0 {
+			logger.Error("Warning: The standard parity is set to 0. This can lead to data loss.")
+		}
 	}()
 
 	region := globalSite.Region

cmd/server_test.go

@@ -59,12 +59,14 @@ type check struct {
 
 // Assert - checks if gotValue is same as expectedValue, if not fails the test.
 func (c *check) Assert(gotValue interface{}, expectedValue interface{}) {
+	c.Helper()
 	if !reflect.DeepEqual(gotValue, expectedValue) {
-		c.Fatalf("Test %s:%s expected %v, got %v", getSource(2), c.testType, expectedValue, gotValue)
+		c.Fatalf("Test %s expected %v, got %v", c.testType, expectedValue, gotValue)
 	}
 }
 
 func verifyError(c *check, response *http.Response, code, description string, statusCode int) {
+	c.Helper()
 	data, err := io.ReadAll(response.Body)
 	c.Assert(err, nil)
 	errorResponse := APIErrorResponse{}

cmd/site-replication.go

@@ -1185,6 +1185,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 			secretKey:     change.Create.SecretKey,
 			sessionPolicy: sp,
 			claims:        change.Create.Claims,
+			comment:       change.Create.Comment,
 		}
 		_, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
 		if err != nil {
@@ -1209,6 +1210,7 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 		opts := updateServiceAccountOpts{
 			secretKey:     change.Update.SecretKey,
 			status:        change.Update.Status,
+			comment:       change.Update.Comment,
 			sessionPolicy: sp,
 		}
 
@@ -1870,6 +1872,7 @@ func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context) error {
 						Claims:        claims,
 						SessionPolicy: json.RawMessage(policyJSON),
 						Status:        acc.Credentials.Status,
+						Comment:       acc.Credentials.Comment,
 					},
 				},
 				UpdatedAt: acc.UpdatedAt,
@@ -4706,6 +4709,7 @@ func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer,
 						Claims:        claims,
 						SessionPolicy: json.RawMessage(policyJSON),
 						Status:        creds.Status,
+						Comment:       creds.Comment,
 					},
 				},
 				UpdatedAt: lastUpdate,

docs/bucket/replication/README.md

@@ -18,14 +18,7 @@ To replicate objects in a bucket to a destination bucket on a target site either
 
 Ensure that versioning is enabled on the source and target buckets with `mc version` command. If object locking is required, the buckets should have been created with `mc mb --with-lock`
 
-Create a replication target on the source cluster as shown below:
-
-```
-mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1
-Remote ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket'
-```
-
-> The user running the above command needs *s3:GetReplicationConfiguration* and *s3:GetBucketVersioning* permission on the source cluster. We do not recommend running root credentials/super admin with replication, instead create a dedicated user. The access credentials used at the destination requires *s3:ReplicateObject* permission.
+The user setting up replication needs *s3:GetReplicationConfiguration* and *s3:GetBucketVersioning* permission on the source cluster. We do not recommend running root credentials/super admin with replication, instead create a dedicated user. The access credentials used at the destination requires *s3:ReplicateObject* permission.
 
 The following minimal permission policy is needed by admin user setting up replication on the `source`:
 
@@ -103,14 +96,24 @@ The access key provided for the replication *target* cluster should have these m
 
 Please note that the permissions required by the admin user on the target cluster can be more fine grained to exclude permissions like "s3:ReplicateDelete", "s3:GetBucketObjectLockConfiguration" etc depending on whether delete replication rules are set up or if object locking is disabled on `destbucket`. The above policies assume that replication of objects, tags and delete marker replication are all enabled on object lock enabled buckets. A sample script to setup replication is provided [here](https://github.com/minio/minio/blob/master/docs/bucket/replication/setup_replication.sh)
 
-Once successfully created and authorized, the `mc admin bucket remote add` command generates a replication target ARN.  This command lists all the currently authorized replication targets:
+To set up replication from a source bucket `srcbucket` on myminio cluster to a bucket `destbucket` on the target minio cluster with endpoint https://replica-endpoint:9000, use:
+```
+mc replicate add myminio/srcbucket --priority 1 --remote-bucket https://accessKey:secretKey@replica-endpoint:9000/destbucket 
+Replication configuration applied successfully to myminio/srcbucket.
+```
+Internally, this creates an ARN for the remote target associating the remote bucket as a replication target to the srcbucket on myminio.By default, if --replicate flag is not specified, replication of delete marker, permanent deletes, existing object replication and replica modification sync are all enabled. If you are using older mc versions, the ARN needs to be generated as a separate step before adding a replication rule.
+
+> NOTE: If you are using a mc version below `RELEASE.2022-12-24T15-21-38Z`, the --remote-bucket flag needs an ARN generated by `mc admin bucket remote add` command. For  mc versions RELEASE.2021-09-02T09-21-27Z and older, the remote target ARN needs to be passed in the --arn flag and actual remote bucket name in --remote-bucket flag of  `mc replicate add`. For example, in older releases of mc replication configuration used to be added with:
 
 ```
-mc admin bucket remote ls myminio/srcbucket --service "replication"
+mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1
 Remote ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket'
+
+mc replicate add myminio/srcbucket/Tax --priority 1 --remote-bucket destbucket --remote-bucket "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --replicate "delete,delete-marker"
+Replication configuration applied successfully to myminio/srcbucket.
 ```
 
-The replication configuration can now be added to the source bucket by applying the json file with replication configuration. The Remote ARN above is passed in as a json element in the configuration.
+The replication configuration generated has the following format and can be exported with `mc replicate export` command:
 
 ```json
 {
@@ -197,18 +200,13 @@ To add a replication rule allowing both delete marker replication, versioned del
 Additional permission of "s3:ReplicateDelete" action would need to be specified on the access key configured for the target cluster if Delete Marker replication or versioned delete replication is enabled.
 
 ```
-mc replicate add myminio/srcbucket/Tax --priority 1 --remote-bucket "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --replicate "delete,delete-marker"
+mc replicate add myminio/srcbucket/Tax --priority 1 --remote-bucket `remote-target` --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --replicate "delete,delete-marker"
 Replication configuration applied successfully to myminio/srcbucket.
 ```
 
-> NOTE: In mc versions RELEASE.2021-09-02T09-21-27Z and older, the remote target ARN needs to be passed in the --arn flag and actual remote bucket name in --remote-bucket flag of  `mc replicate add`. For example, with the ARN above the replication configuration used to be added with
+> NOTE: In mc versions `RELEASE.2022-12-24T15-21-38Z` and above `remote-target`  should be of the format `https://accessKey:secretKey@replica-endpoint:9000/destbucket` which earlier used to be set during `mc admin bucket remote add`. For older releases, use the arn generated with `mc admin bucket remote add` command - e.g."arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" as the `remote-target`.
 
-```
-mc replicate add myminio/srcbucket/Tax --priority 1 --remote-bucket destbucket --arn "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --replicate "delete,delete-marker"
-Replication configuration applied successfully to myminio/srcbucket.
-```
-
-Also note that for `mc` version `RELEASE.2021-09-02T09-21-27Z` or older supports only a single remote target per bucket. To take advantage of multiple destination replication, use the latest version of `mc`
+Also note that `mc` version `RELEASE.2021-09-02T09-21-27Z` or older supports only a single remote target per bucket. To take advantage of multiple destination replication, use the latest version of `mc`
 
 Status of delete marker replication can be viewed by doing a GET/HEAD on the object version - it will return a `X-Minio-Replication-DeleteMarker-Status` header and http response code of `405`. In the case of permanent deletes, if the delete replication is pending or failed to propagate to the target cluster, GET/HEAD will return additional `X-Minio-Replication-Delete-Status` header and a http response code of `405`.
 
@@ -221,7 +219,8 @@ On the target bucket, `s3:PutObject` event shows `X-Amz-Replication-Status` stat
 ### Sync/Async Replication
 
 By default, replication is completed asynchronously. If synchronous replication is desired, set the --sync flag while adding a
-remote replication target using the `mc admin bucket remote add` command
+remote replication target using the `mc admin bucket remote add` command. For mc releases on or after `RELEASE.2022-12-24T15-21-38Z`, the 
+--sync, --health-check and --bandwidth flags can be specified in `mc replicate add|update` command
 
 ```
  mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1 --sync --healthcheck-seconds 100

docs/bucket/replication/delete-replication.sh

@@ -62,8 +62,7 @@ export MC_HOST_myminio2=http://minioadmin:minioadmin@localhost:9002
 ./mc mb myminio2/testbucket/
 ./mc version enable myminio2/testbucket/
 
-arn=$(mc admin bucket remote add myminio1/testbucket/ http://minioadmin:minioadmin@localhost:9002/testbucket/ --service "replication" --json | jq -r .RemoteARN)
-./mc replicate add myminio1/testbucket --remote-bucket "$arn" --priority 1
+./mc replicate add myminio1/testbucket --remote-bucket http://minioadmin:minioadmin@localhost:9002/testbucket/ --priority 1
 
 ./mc cp README.md myminio1/testbucket/dir/file
 ./mc cp README.md myminio1/testbucket/dir/file

docs/bucket/replication/setup_2site_existing_replication.sh

@@ -76,14 +76,11 @@ done
 ./mc mb siteb/bucket/
 ./mc version enable siteb/bucket/
 
-echo "adding replication config for site a -> site b"
-remote_arn=$(./mc admin bucket remote add sitea/bucket/ \
-   http://minio:minio123@127.0.0.1:9004/bucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-echo "adding replication rule for a -> b : ${remote_arn}"
-
+echo "adding replication rule for site a -> site b"
 ./mc replicate add sitea/bucket/ \
-   --remote-bucket "${remote_arn}"
+     --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket
+
+remote_arn=$(./mc replicate ls sitea/bucket --json | jq -r .rule.Destination.Bucket)
 sleep 1
 
 ./mc replicate resync start sitea/bucket/ --remote-bucket "${remote_arn}"

docs/bucket/replication/setup_3site_replication.sh

@@ -80,127 +80,76 @@ export MC_HOST_sitec=http://minio:minio123@127.0.0.1:9006
 ./mc version enable sitec/bucket/
 ./mc mb -l sitec/olockbucket
 
-echo "adding replication config for site a -> site b"
-remote_arn=$(./mc admin bucket remote add sitea/bucket/ \
-   http://minio:minio123@127.0.0.1:9004/bucket \
-   --service "replication" --json | jq -r ".RemoteARN")
 echo "adding replication rule for a -> b : ${remote_arn}"
 sleep 1
 ./mc replicate add sitea/bucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
 sleep 1
 
-echo "adding replication config for site b -> site a"
-remote_arn=$(./mc admin bucket remote add siteb/bucket/ \
-   http://minio:minio123@127.0.0.1:9001/bucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
 echo "adding replication rule for b -> a : ${remote_arn}"
 ./mc replicate add siteb/bucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
 sleep 1
 
-echo "adding replication config for site a -> site c"
-remote_arn=$(./mc admin bucket remote add sitea/bucket/ \
-   http://minio:minio123@127.0.0.1:9006/bucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
 echo "adding replication rule for a -> c : ${remote_arn}"
 ./mc replicate add sitea/bucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
 sleep 1
-echo "adding replication config for site c -> site a"
-remote_arn=$(./mc admin bucket remote add sitec/bucket/ \
-   http://minio:minio123@127.0.0.1:9001/bucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for c -> a : ${remote_arn}"
 ./mc replicate add sitec/bucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
 sleep 1
-echo "adding replication config for site b -> site c"
-remote_arn=$(./mc admin bucket remote add siteb/bucket/ \
-   http://minio:minio123@127.0.0.1:9006/bucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for b -> c : ${remote_arn}"
 ./mc replicate add siteb/bucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
 sleep 1
 
-echo "adding replication config for site c -> site b"
-remote_arn=$(././mc admin bucket remote add sitec/bucket \
-   http://minio:minio123@127.0.0.1:9004/bucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
 echo "adding replication rule for c -> b : ${remote_arn}"
 ./mc replicate add sitec/bucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
 sleep 1
-echo "adding replication config for olockbucket site a -> site b"
-remote_arn=$(././mc admin bucket remote add sitea/olockbucket/ \
-   http://minio:minio123@127.0.0.1:9004/olockbucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for olockbucket a -> b : ${remote_arn}"
 ./mc replicate add sitea/olockbucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
 sleep 1
-echo "adding replication config for site b -> site a"
-remote_arn=$(././mc admin bucket remote add siteb/olockbucket/ \
-   http://minio:minio123@127.0.0.1:9001/olockbucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for olockbucket b -> a : ${remote_arn}"
 ./mc replicate add siteb/olockbucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync"
 sleep 1
-echo "adding replication config for olockbucket site a -> site c"
-remote_arn=$(././mc admin bucket remote add sitea/olockbucket/ \
-   http://minio:minio123@127.0.0.1:9006/olockbucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for olockbucket a -> c : ${remote_arn}"
 ./mc replicate add sitea/olockbucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
 sleep 1
-echo "adding replication config for site c -> site a"
-remote_arn=$(././mc admin bucket remote add sitec/olockbucket/ \
-   http://minio:minio123@127.0.0.1:9001/olockbucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for olockbucket c -> a : ${remote_arn}"
 ./mc replicate add sitec/olockbucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2
 sleep 1
-echo "adding replication config for site b -> site c"
-remote_arn=$(././mc admin bucket remote add siteb/olockbucket/ \
-   http://minio:minio123@127.0.0.1:9006/olockbucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for olockbucket b -> c : ${remote_arn}"
 ./mc replicate add siteb/olockbucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
 sleep 1
-echo "adding replication config for site c -> site b"
-remote_arn=$(././mc admin bucket remote add sitec/olockbucket \
-   http://minio:minio123@127.0.0.1:9004/olockbucket \
-   --service "replication" --json | jq -r ".RemoteARN")
-sleep 1
+
 echo "adding replication rule for olockbucket c -> b : ${remote_arn}"
 ./mc replicate add sitec/olockbucket/ \
-   --remote-bucket "${remote_arn}" \
+   --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \
    --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3
 sleep 1
 

docs/bucket/replication/setup_replication.sh

@@ -96,10 +96,6 @@ cat ./replpolicy.json
 # assign this replication policy to repluser
 mc admin policy set dest replpolicy user=repluser
 
-# define remote target for replication from source/bucket -> dest/bucket
-remote_arn=$(mc admin bucket remote add repladminAlias/bucket http://repluser:repluser123@localhost:9000/bucket --service replication --json | jq -r ".RemoteARN")
-
-echo "Now, use this ARN to add replication rules using 'mc replicate add' command"
-# use arn returned by above command to create a replication policy on the source/bucket with `mc replicate add`
-mc replicate add source/bucket --priority 1 --remote-bucket "${remote_arn}" \
+# configure replication config to remote bucket at http://localhost:9000
+mc replicate add source/bucket --priority 1 --remote-bucket http://repluser:repluser123@localhost:9000/bucket \
    --replicate existing-objects,delete,delete-marker,replica-metadata-sync

docs/deployment/kernel-tuning/README.md

@@ -1,98 +0,0 @@
-# Kernel Tuning for MinIO Production Deployment on Linux Servers [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
-
-List of kernel tuning recommendations for MinIO server deployments, a copy of this [script](https://github.com/minio/minio/blob/master/docs/deployment/kernel-tuning/sysctl.sh) is available here that can be applied on the servers.
-
-> NOTE: These are common recommendations on any Linux servers, users must be careful on any premature optimization.
-> These settings are not mandatory, these settings do not fix any hardware issues and should not be considered as an
-> alternative to boost performance to mask underlying hardware problems. Under all circumstances it is recommended
-> to perform this tuning only after performing benchmarks for the hardware with expected baseline results.
-
-```
-#!/bin/bash
-
-cat > sysctl.conf <<EOF
-# maximum number of open files/file descriptors
-fs.file-max = 4194303
-
-# use as little swap space as possible
-vm.swappiness = 1
-
-# prioritize application RAM against disk/swap cache
-vm.vfs_cache_pressure = 50
-
-# minimum free memory
-vm.min_free_kbytes = 1000000
-
-# follow mellanox best practices https://community.mellanox.com/s/article/linux-sysctl-tuning
-# the following changes are recommended for improving IPv4 traffic performance by Mellanox
-
-# disable the TCP timestamps option for better CPU utilization
-net.ipv4.tcp_timestamps = 0
-
-# enable the TCP selective acks option for better throughput
-net.ipv4.tcp_sack = 1
-
-# increase the maximum length of processor input queues
-net.core.netdev_max_backlog = 250000
-
-# increase the TCP maximum and default buffer sizes using setsockopt()
-net.core.rmem_max = 4194304
-net.core.wmem_max = 4194304
-net.core.rmem_default = 4194304
-net.core.wmem_default = 4194304
-net.core.optmem_max = 4194304
-
-# increase memory thresholds to prevent packet dropping:
-net.ipv4.tcp_rmem = "4096 87380 4194304"
-net.ipv4.tcp_wmem = "4096 65536 4194304"
-
-# enable low latency mode for TCP:
-net.ipv4.tcp_low_latency = 1
-
-# the following variable is used to tell the kernel how much of the socket buffer
-# space should be used for TCP window size, and how much to save for an application
-# buffer. A value of 1 means the socket buffer will be divided evenly between.
-# TCP windows size and application.
-net.ipv4.tcp_adv_win_scale = 1
-
-# maximum number of incoming connections
-net.core.somaxconn = 65535
-
-# maximum number of packets queued
-net.core.netdev_max_backlog = 10000
-
-# queue length of completely established sockets waiting for accept
-net.ipv4.tcp_max_syn_backlog = 4096
-
-# time to wait (seconds) for FIN packet
-net.ipv4.tcp_fin_timeout = 15
-
-# disable icmp send redirects
-net.ipv4.conf.all.send_redirects = 0
-
-# disable icmp accept redirect
-net.ipv4.conf.all.accept_redirects = 0
-
-# drop packets with LSR or SSR
-net.ipv4.conf.all.accept_source_route = 0
-
-# MTU discovery, only enable when ICMP blackhole detected
-net.ipv4.tcp_mtu_probing = 1
-
-EOF
-
-echo "Enabling system level tuning params"
-sysctl --quiet --load sysctl.conf && rm -f sysctl.conf
-
-# `Transparent Hugepage Support`*: This is a Linux kernel feature intended to improve
-# performance by making more efficient use of processor’s memory-mapping hardware.
-# But this may cause https://blogs.oracle.com/linux/performance-issues-with-transparent-huge-pages-thp
-# for non-optimized applications. As most Linux distributions set it to `enabled=always` by default,
-# we recommend changing this to `enabled=madvise`. This will allow applications optimized
-# for transparent hugepages to obtain the performance benefits, while preventing the
-# associated problems otherwise. Also, set `transparent_hugepage=madvise` on your kernel
-# command line (e.g. in /etc/default/grub) to persistently set this value.
-
-echo "Enabling THP madvise"
-echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled
-```

docs/deployment/kernel-tuning/sysctl.sh

@@ -1,87 +0,0 @@
-#!/bin/bash
-
-cat > sysctl.conf <<EOF
-# maximum number of open files/file descriptors
-fs.file-max = 4194303
-
-# use as little swap space as possible
-vm.swappiness = 1
-
-# prioritize application RAM against disk/swap cache
-vm.vfs_cache_pressure = 50
-
-# minimum free memory
-vm.min_free_kbytes = 1000000
-
-# follow mellanox best practices https://community.mellanox.com/s/article/linux-sysctl-tuning
-# the following changes are recommended for improving IPv4 traffic performance by Mellanox
-
-# disable the TCP timestamps option for better CPU utilization
-net.ipv4.tcp_timestamps = 0
-
-# enable the TCP selective acks option for better throughput
-net.ipv4.tcp_sack = 1
-
-# increase the maximum length of processor input queues
-net.core.netdev_max_backlog = 250000
-
-# increase the TCP maximum and default buffer sizes using setsockopt()
-net.core.rmem_max = 4194304
-net.core.wmem_max = 4194304
-net.core.rmem_default = 4194304
-net.core.wmem_default = 4194304
-net.core.optmem_max = 4194304
-
-# increase memory thresholds to prevent packet dropping:
-net.ipv4.tcp_rmem = 4096 87380 4194304
-net.ipv4.tcp_wmem = 4096 65536 4194304
-
-# enable low latency mode for TCP:
-net.ipv4.tcp_low_latency = 1
-
-# the following variable is used to tell the kernel how much of the socket buffer
-# space should be used for TCP window size, and how much to save for an application
-# buffer. A value of 1 means the socket buffer will be divided evenly between.
-# TCP windows size and application.
-net.ipv4.tcp_adv_win_scale = 1
-
-# maximum number of incoming connections
-net.core.somaxconn = 65535
-
-# maximum number of packets queued
-net.core.netdev_max_backlog = 10000
-
-# queue length of completely established sockets waiting for accept
-net.ipv4.tcp_max_syn_backlog = 4096
-
-# time to wait (seconds) for FIN packet
-net.ipv4.tcp_fin_timeout = 15
-
-# disable icmp send redirects
-net.ipv4.conf.all.send_redirects = 0
-
-# disable icmp accept redirect
-net.ipv4.conf.all.accept_redirects = 0
-
-# drop packets with LSR or SSR
-net.ipv4.conf.all.accept_source_route = 0
-
-# MTU discovery, only enable when ICMP blackhole detected
-net.ipv4.tcp_mtu_probing = 1
-
-EOF
-
-echo "Enabling system level tuning params"
-sysctl --quiet --load sysctl.conf && rm -f sysctl.conf
-
-# `Transparent Hugepage Support`*: This is a Linux kernel feature intended to improve
-# performance by making more efficient use of processor’s memory-mapping hardware.
-# But this may cause https://blogs.oracle.com/linux/performance-issues-with-transparent-huge-pages-thp
-# for non-optimized applications. As most Linux distributions set it to `enabled=always` by default,
-# we recommend changing this to `enabled=madvise`. This will allow applications optimized
-# for transparent hugepages to obtain the performance benefits, while preventing the
-# associated problems otherwise. Also, set `transparent_hugepage=madvise` on your kernel
-# command line (e.g. in /etc/default/grub) to persistently set this value.
-
-echo "Enabling THP madvise"
-echo madvise | sudo tee /sys/kernel/mm/transparent_hugepage/enabled

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-12-12T19-27-27Z
+  image: quay.io/minio/minio:RELEASE.2023-01-06T18-11-18Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/inconshreveable/mousetrap v1.1.0
 	github.com/json-iterator/go v1.1.12
-	github.com/klauspost/compress v1.15.13
+	github.com/klauspost/compress v1.15.14
 	github.com/klauspost/cpuid/v2 v2.2.3
 	github.com/klauspost/filepathx v1.1.1
 	github.com/klauspost/pgzip v1.2.5
@@ -44,12 +44,12 @@ require (
 	github.com/lithammer/shortuuid/v4 v4.0.0
 	github.com/miekg/dns v1.1.50
 	github.com/minio/cli v1.24.2
-	github.com/minio/console v0.22.3
+	github.com/minio/console v0.22.5
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/dperf v0.4.2
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.22.2
-	github.com/minio/madmin-go/v2 v2.0.3
+	github.com/minio/madmin-go/v2 v2.0.6
 	github.com/minio/minio-go/v7 v7.0.45
 	github.com/minio/pkg v1.5.8
 	github.com/minio/selfupdate v0.5.0
@@ -84,9 +84,9 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.6
 	go.uber.org/atomic v1.10.0
 	go.uber.org/zap v1.24.0
-	golang.org/x/crypto v0.4.0
-	golang.org/x/oauth2 v0.3.0
-	golang.org/x/sys v0.3.0
+	golang.org/x/crypto v0.5.0
+	golang.org/x/oauth2 v0.4.0
+	golang.org/x/sys v0.4.0
 	golang.org/x/time v0.3.0
 	google.golang.org/api v0.105.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -94,8 +94,6 @@ require (
 
 replace github.com/mattn/go-ieproxy => github.com/mattn/go-ieproxy v0.0.1 // do not remove until https://github.com/mattn/go-ieproxy/issues/45 is fixed
 
-replace github.com/klauspost/compress => github.com/harshavardhana/compress v1.15.14-0.20221229182712-881121b1ca7f
-
 require (
 	aead.dev/mem v0.2.0 // indirect
 	cloud.google.com/go v0.107.0 // indirect
@@ -219,10 +217,10 @@ require (
 	go.opencensus.io v0.24.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.4.0 // indirect
+	golang.org/x/net v0.5.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/term v0.3.0 // indirect
-	golang.org/x/text v0.5.0 // indirect
+	golang.org/x/term v0.4.0 // indirect
+	golang.org/x/text v0.6.0 // indirect
 	golang.org/x/tools v0.4.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect

go.sum

@@ -194,7 +194,7 @@ github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.28 h1:ndAExarwr5Y+GaHE6VCaY1kyS/HwwGGyuimVhWsHOEM=
 github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
-github.com/Azure/go-autorest/autorest/adal v0.9.20 h1:gJ3E98kMpFB1MFqQCvA1yFab8vthOeD4VlFRQULxahg=
+github.com/Azure/go-autorest/autorest/adal v0.9.18 h1:kLnPsRjzZZUF3K5REu/Kc+qMQrvuza2bwSnNdhmzLfQ=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
@@ -570,8 +570,6 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/harshavardhana/compress v1.15.14-0.20221229182712-881121b1ca7f h1:YnJFZ1tFSWJV914TdGqssNrjkrFOf/2fxuMiBfOEoDM=
-github.com/harshavardhana/compress v1.15.14-0.20221229182712-881121b1ca7f/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -655,6 +653,11 @@ github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaR
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
+github.com/klauspost/compress v1.15.14 h1:i7WCKDToww0wA+9qrUZ1xOjp218vfFo3nTU6UHp+gOc=
+github.com/klauspost/compress v1.15.14/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM=
 github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
@@ -753,8 +756,8 @@ github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg=
 github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.4 h1:sNJYTb2uNswdqmGARg9wrogCX8+GRZzEacYbJT86e00=
 github.com/minio/colorjson v1.0.4/go.mod h1:ZgE8vYon4xC4yfBPclP/2gqMRYw+p+xRsBbLMDKdb9M=
-github.com/minio/console v0.22.3 h1:udCmeWr3c18PqXNDLMfCy+HMU3/WPocHjKZUYBPQodM=
-github.com/minio/console v0.22.3/go.mod h1:eoDU1YYpzH0zZXIhDnqhktAQG+BNn/mN9QN87gtWOLY=
+github.com/minio/console v0.22.5 h1:Z4sKPgoms0HUoWqKbmR/cJCndwaPVug5zPehZFDBTOE=
+github.com/minio/console v0.22.5/go.mod h1:M82OalYqrzGbHVGNShD4Qx/XbXYY6uLohk300n7UqE4=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/dperf v0.4.2 h1:u2fcdXViX3+alZcEYSOh1crNbGtVU5drQ24lP3Xx6j8=
@@ -767,8 +770,8 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
 github.com/minio/kes v0.22.2 h1:9NdgTx+TFJco0Pqdrq8WZbrTZVv0ichg+sbPRQiJ2HU=
 github.com/minio/kes v0.22.2/go.mod h1:J9sD6Pe8obPt7+JXFcznkWaYaj9pBWCfN9U9j//NsNw=
 github.com/minio/madmin-go v1.6.6/go.mod h1:ATvkBOLiP3av4D++2v1UEHC/QzsGtgXD5kYvvRYzdKs=
-github.com/minio/madmin-go/v2 v2.0.3 h1:Q8qco+JrbRIim25tGrs0enVRJGoIMUHfULa5nJoSiqM=
-github.com/minio/madmin-go/v2 v2.0.3/go.mod h1:5aFi/VLWBHC2DEFfGIlUmAeJhaF4ZAjuYpEWZFU14Zw=
+github.com/minio/madmin-go/v2 v2.0.6 h1:d0cfiH5SkC8vZHgRtcki8j37fb3FF65cTdjUdfBR8ks=
+github.com/minio/madmin-go/v2 v2.0.6/go.mod h1:5aFi/VLWBHC2DEFfGIlUmAeJhaF4ZAjuYpEWZFU14Zw=
 github.com/minio/mc v0.0.0-20221224152138-176072dee43d h1:etzZIWQ3NFrxzwnvjczETWMcgoja9ZKLFLIfQzvpqP8=
 github.com/minio/mc v0.0.0-20221224152138-176072dee43d/go.mod h1:af4hDQUHwu8az+6TyEKXa2Yd+lvMDVgnc9/kstHPZY8=
 github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
@@ -1122,8 +1125,8 @@ golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20221012134737-56aed061732a/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.4.0 h1:UVQgzMY87xqpKNgb+kDsll2Igd33HszWHFLmpaRMq/8=
-golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80=
+golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
+golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1220,8 +1223,8 @@ golang.org/x/net v0.0.0-20220725212005-46097bf591d3/go.mod h1:AaygXjzTFtRAg2ttMY
 golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.0.0-20221017152216-f25eb7ecb193/go.mod h1:RpDiru2p0u2F0lLpEoqnP2+7xs0ifAuOcJ442g6GU2s=
 golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
-golang.org/x/net v0.4.0 h1:Q5QPcMlvfxFTAPV0+07Xz/MpK9NTXu2VDUuy0FeMfaU=
-golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
+golang.org/x/net v0.5.0 h1:GyT4nK/YDHSqa1c4753ouYCDajOYKTja9Xb/OHtgvSw=
+golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1246,8 +1249,8 @@ golang.org/x/oauth2 v0.0.0-20220608161450-d0670ef3b1eb/go.mod h1:jaDAt6Dkxork7Lm
 golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7LmZnYtzbRWj0W47D86a3TGe0YHBvmE=
 golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
 golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg=
-golang.org/x/oauth2 v0.3.0 h1:6l90koy8/LaBLmLu8jpHeHexzMwEita0zFfYlggy2F8=
-golang.org/x/oauth2 v0.3.0/go.mod h1:rQrIauxkUhJ6CuwEXwymO2/eh4xz2ZWF1nBkcxS+tGk=
+golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
+golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -1365,13 +1368,15 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20201210144234-2321bbc49cbf/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
 golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
+golang.org/x/term v0.4.0 h1:O7UWfv5+A2qiuulQk30kVinPoMtoIPeVaKLEgLpVkvg=
+golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1382,8 +1387,9 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.5.0 h1:OLmvp0KP+FVG99Ct/qFiL/Fhk4zp4QQnZ7b2U+5piUM=
 golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.6.0 h1:3XmdazWV+ubf7QgHSTWeykHOci5oeekaGJBLkrkaw4k=
+golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

internal/auth/credentials.go

@@ -102,6 +102,7 @@ type Credentials struct {
 	ParentUser   string                 `xml:"-" json:"parentUser,omitempty"`
 	Groups       []string               `xml:"-" json:"groups,omitempty"`
 	Claims       map[string]interface{} `xml:"-" json:"claims,omitempty"`
+	Comment      string                 `xml:"-" json:"comment,omitempty"`
 }
 
 func (cred Credentials) String() string {

internal/config/storageclass/storage-class.go

@@ -304,12 +304,7 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
 	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {
-		cfg.RRS.Parity = defaultRRSParity
-		if setDriveCount == 1 {
-			cfg.RRS.Parity = 0
-		}
-		cfg.Standard.Parity = DefaultParityBlocks(setDriveCount)
-		return cfg, err
+		return Config{}, err
 	}
 
 	return cfg, nil

internal/http/dial_others.go

@@ -28,7 +28,8 @@ import (
 )
 
 // TODO: if possible implement for non-linux platforms, not a priority at the moment
-//nolint:deadcode
+//
+//nolint:unused
 func setTCPParameters(string, string, syscall.RawConn) error {
 	return nil
 }

internal/http/listener.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"strings"
 	"syscall"
 )
 
@@ -136,20 +137,49 @@ func newHTTPListener(ctx context.Context, serverAddrs []string) (listener *httpL
 		}
 	}()
 
+	isLocalhost := false
+	for _, serverAddr := range serverAddrs {
+		host, _, err := net.SplitHostPort(serverAddr)
+		if err == nil {
+			if strings.EqualFold(host, "localhost") {
+				isLocalhost = true
+			}
+		}
+	}
+
+	// Silently ignore failure to bind on DNS cached ipv6 loopback iff user specifies "localhost"
 	for _, serverAddr := range serverAddrs {
 		var l net.Listener
 		if l, err = listenCfg.Listen(ctx, "tcp", serverAddr); err != nil {
+			if isLocalhost && strings.HasPrefix(serverAddr, "[::1]") {
+				continue
+			}
 			return nil, err
 		}
 
 		tcpListener, ok := l.(*net.TCPListener)
 		if !ok {
-			return nil, fmt.Errorf("unexpected listener type found %v, expected net.TCPListener", l)
+			err = fmt.Errorf("unexpected listener type found %v, expected net.TCPListener", l)
+			if isLocalhost && strings.HasPrefix(serverAddr, "[::1]") {
+				continue
+			}
+			return nil, err
 		}
 
 		tcpListeners = append(tcpListeners, tcpListener)
 	}
 
+	// Fail if no listeners found
+	if len(tcpListeners) == 0 {
+		// Report specific issue
+		if err != nil {
+			return nil, err
+		}
+		// Report general issue
+		err = fmt.Errorf("%v listeners found, expected at least 1", 0)
+		return nil, err
+	}
+
 	listener = &httpListener{
 		tcpListeners: tcpListeners,
 		acceptCh:     make(chan acceptResult, len(tcpListeners)),

internal/http/listener_test.go

@@ -142,17 +142,18 @@ func TestNewHTTPListener(t *testing.T) {
 		{[]string{"example.org:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
 		{[]string{"unknown-host"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
 		{[]string{"unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
+		{[]string{"localhost:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
 		{[]string{"localhost:65432", "93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
 		{[]string{"localhost:65432", "unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), true},
 		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
 		{[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
+		{[]string{"[::1]:9090", "localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), false},
 	}
 
 	for _, testCase := range testCases {
 		listener, err := newHTTPListener(context.Background(),
 			testCase.serverAddrs,
 		)
-
 		if !testCase.expectedErr {
 			if err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)