treat all localhost endpoints as local setup with same port (#18784)

fixes #18783 and avoids user mistakes

cmd/bucket-replication.go

@@ -2255,7 +2255,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 		if rs != nil {
 			h, err := rs.ToHeader()
 			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				logger.LogIf(ctx, fmt.Errorf("invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
 				continue
 			}
 			gopts.Set(xhttp.Range, h)
@@ -2348,6 +2348,127 @@ func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc
 	}
 }
 
+// proxyTaggingToRepTarget proxies tagging requests to remote targets for
+// active-active replicated setups
+func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *tags.Tags, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (proxy proxyResult) {
+	// this option is set when active-active replication is in place between site A -> B,
+	// and request hits site B that does not have the object yet.
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
+		return proxy
+	}
+	var wg sync.WaitGroup
+	errs := make([]error, len(proxyTargets.Targets))
+	for idx, t := range proxyTargets.Targets {
+		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
+		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			continue
+		}
+		// if proxying explicitly disabled on remote target
+		if tgt.disableProxy {
+			continue
+		}
+		idx := idx
+		wg.Add(1)
+		go func(idx int, tgt *TargetClient) {
+			defer wg.Done()
+			var err error
+			if tags != nil {
+				popts := minio.PutObjectTaggingOptions{
+					VersionID: opts.VersionID,
+					Internal: minio.AdvancedObjectTaggingOptions{
+						ReplicationProxyRequest: "true",
+					},
+				}
+				err = tgt.PutObjectTagging(ctx, tgt.Bucket, object, tags, popts)
+			} else {
+				dopts := minio.RemoveObjectTaggingOptions{
+					VersionID: opts.VersionID,
+					Internal: minio.AdvancedObjectTaggingOptions{
+						ReplicationProxyRequest: "true",
+					},
+				}
+				err = tgt.RemoveObjectTagging(ctx, tgt.Bucket, object, dopts)
+			}
+			if err != nil {
+				errs[idx] = err
+			}
+		}(idx, tgt)
+	}
+	wg.Wait()
+
+	var (
+		terr        error
+		taggedCount int
+	)
+	for _, err := range errs {
+		if err == nil {
+			taggedCount++
+			continue
+		}
+		errCode := minio.ToErrorResponse(err).Code
+		if errCode != "NoSuchKey" && errCode != "NoSuchVersion" {
+			terr = err
+		}
+	}
+	// don't return error if at least one target was tagged successfully
+	if taggedCount == 0 && terr != nil {
+		proxy.Err = terr
+	}
+	return proxy
+}
+
+// proxyGetTaggingToRepTarget proxies get tagging requests to remote targets for
+// active-active replicated setups
+func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgs *tags.Tags, proxy proxyResult) {
+	// this option is set when active-active replication is in place between site A -> B,
+	// and request hits site B that does not have the object yet.
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
+		return nil, proxy
+	}
+	var wg sync.WaitGroup
+	errs := make([]error, len(proxyTargets.Targets))
+	tagSlc := make([]map[string]string, len(proxyTargets.Targets))
+	for idx, t := range proxyTargets.Targets {
+		tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
+		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			continue
+		}
+		// if proxying explicitly disabled on remote target
+		if tgt.disableProxy {
+			continue
+		}
+		idx := idx
+		wg.Add(1)
+		go func(idx int, tgt *TargetClient) {
+			defer wg.Done()
+			var err error
+			gopts := minio.GetObjectTaggingOptions{
+				VersionID: opts.VersionID,
+				Internal: minio.AdvancedObjectTaggingOptions{
+					ReplicationProxyRequest: "true",
+				},
+			}
+			tgs, err = tgt.GetObjectTagging(ctx, tgt.Bucket, object, gopts)
+			if err != nil {
+				errs[idx] = err
+			} else {
+				tagSlc[idx] = tgs.ToMap()
+			}
+		}(idx, tgt)
+	}
+	wg.Wait()
+	for idx, err := range errs {
+		errCode := minio.ToErrorResponse(err).Code
+		if err != nil && errCode != "NoSuchKey" && errCode != "NoSuchVersion" {
+			return nil, proxyResult{Err: err}
+		}
+		if err == nil {
+			tgs, _ = tags.MapToObjectTags(tagSlc[idx])
+		}
+	}
+	return tgs, proxy
+}
+
 func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer) {
 	globalReplicationPool.queueReplicaDeleteTask(dv)
 	for arn := range dv.ReplicationState.Targets {

cmd/bucket-targets.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"sync"
@@ -351,6 +352,27 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
 		}
 	}
+
+	// Check if target is a MinIO server and alive
+	hcCtx, cancel := context.WithTimeout(ctx, 3*time.Second)
+	scheme := "http"
+	if tgt.Secure {
+		scheme = "https"
+	}
+	result := <-sys.hcClient.Alive(hcCtx, madmin.AliveOpts{}, madmin.ServerProperties{
+		Endpoint: tgt.Endpoint,
+		Scheme:   scheme,
+	})
+
+	cancel()
+	if result.Error != nil {
+		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: result.Error, AccessKey: tgt.Credentials.AccessKey}
+	}
+	if !result.Online {
+		err := errors.New("Health check timed out after 3 seconds")
+		return RemoteTargetConnectionErr{Err: err}
+	}
+
 	sys.Lock()
 	defer sys.Unlock()
 

cmd/data-usage-cache.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,6 +32,7 @@ import (
 	"time"
 
 	"github.com/cespare/xxhash/v2"
+	"github.com/dustin/go-humanize"
 	"github.com/klauspost/compress/zstd"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -46,6 +47,10 @@ import (
 // dataUsageHash is the hash type used.
 type dataUsageHash string
 
+// sizeHistogramV1 is size histogram V1, which has fewer intervals esp. between
+// 1024B and 1MiB.
+type sizeHistogramV1 [dataUsageBucketLenV1]uint64
+
 // sizeHistogram is a size histogram.
 type sizeHistogram [dataUsageBucketLen]uint64
 
@@ -204,8 +209,8 @@ func (r *replicationAllStats) clone() *replicationAllStats {
 	return &dst
 }
 
-//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
-//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
+//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
+//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
 
 //msgp:tuple dataUsageEntryV2
 type dataUsageEntryV2 struct {
@@ -263,14 +268,28 @@ type dataUsageEntryV6 struct {
 	Compacted        bool
 }
 
+type dataUsageEntryV7 struct {
+	Children dataUsageHashMap `msg:"ch"`
+	// These fields do no include any children.
+	Size             int64                `msg:"sz"`
+	Objects          uint64               `msg:"os"`
+	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
+	DeleteMarkers    uint64               `msg:"dms"`
+	ObjSizes         sizeHistogramV1      `msg:"szs"`
+	ObjVersions      versionsHistogram    `msg:"vh"`
+	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
+	AllTierStats     *allTierStats        `msg:"ats,omitempty"`
+	Compacted        bool                 `msg:"c"`
+}
+
 // dataUsageCache contains a cache of data usage entries latest version.
 type dataUsageCache struct {
 	Info  dataUsageCacheInfo
 	Cache map[string]dataUsageEntry
 }
 
-//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
-//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6
+//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
+//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
 
 // dataUsageCacheV2 contains a cache of data usage entries version 2.
 type dataUsageCacheV2 struct {
@@ -302,6 +321,12 @@ type dataUsageCacheV6 struct {
 	Cache map[string]dataUsageEntryV6
 }
 
+// dataUsageCacheV7 contains a cache of data usage entries version 7.
+type dataUsageCacheV7 struct {
+	Info  dataUsageCacheInfo
+	Cache map[string]dataUsageEntryV7
+}
+
 //msgp:ignore dataUsageEntryInfo
 type dataUsageEntryInfo struct {
 	Name   string
@@ -750,11 +775,42 @@ func (h *sizeHistogram) add(size int64) {
 	}
 }
 
+// mergeV1 is used to migrate data usage cache from sizeHistogramV1 to
+// sizeHistogram
+func (h *sizeHistogram) mergeV1(v sizeHistogramV1) {
+	var oidx, nidx int
+	for oidx < len(v) {
+		intOld, intNew := ObjectsHistogramIntervalsV1[oidx], ObjectsHistogramIntervals[nidx]
+		// skip intervals that aren't common to both histograms
+		if intOld.start != intNew.start || intOld.end != intNew.end {
+			nidx++
+			continue
+		}
+		h[nidx] += v[oidx]
+		oidx++
+		nidx++
+	}
+}
+
 // toMap returns the map to a map[string]uint64.
 func (h *sizeHistogram) toMap() map[string]uint64 {
 	res := make(map[string]uint64, dataUsageBucketLen)
+	var splCount uint64
 	for i, count := range h {
-		res[ObjectsHistogramIntervals[i].name] = count
+		szInt := ObjectsHistogramIntervals[i]
+		switch {
+		case humanize.KiByte == szInt.start && szInt.end == humanize.MiByte-1:
+			// spl interval: [1024B, 1MiB)
+			res[szInt.name] = splCount
+		case humanize.KiByte <= szInt.start && szInt.end <= humanize.MiByte-1:
+			// intervals that fall within the spl interval above; they
+			// appear earlier in this array of intervals, see
+			// ObjectsHistogramIntervals
+			splCount += count
+			fallthrough
+		default:
+			res[szInt.name] = count
+		}
 	}
 	return res
 }
@@ -1027,7 +1083,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 // Bumping the cache version will drop data from previous versions
 // and write new data with the new version.
 const (
-	dataUsageCacheVerCurrent = 7
+	dataUsageCacheVerCurrent = 8
+	dataUsageCacheVerV7      = 7
 	dataUsageCacheVerV6      = 6
 	dataUsageCacheVerV5      = 5
 	dataUsageCacheVerV4      = 4
@@ -1275,6 +1332,34 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			}
 			d.Cache[k] = due
 		}
+		return nil
+	case dataUsageCacheVerV7:
+		// Zstd compressed.
+		dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
+		if err != nil {
+			return err
+		}
+		defer dec.Close()
+		dold := &dataUsageCacheV7{}
+		if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
+			return err
+		}
+		d.Info = dold.Info
+		d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
+		for k, v := range dold.Cache {
+			var szHist sizeHistogram
+			szHist.mergeV1(v.ObjSizes)
+			d.Cache[k] = dataUsageEntry{
+				Children:         v.Children,
+				Size:             v.Size,
+				Objects:          v.Objects,
+				Versions:         v.Versions,
+				ObjSizes:         szHist,
+				ReplicationStats: v.ReplicationStats,
+				Compacted:        v.Compacted,
+			}
+		}
+
 		return nil
 	case dataUsageCacheVerCurrent:
 		// Zstd compressed.

cmd/data-usage-cache_gen.go

@@ -1480,6 +1480,149 @@ func (z *dataUsageCacheV6) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageCacheV7) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			err = z.Info.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV7, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 dataUsageEntryV7
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				err = za0002.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageCacheV7) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Info":
+			bts, err = z.Info.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Info")
+				return
+			}
+		case "Cache":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Cache")
+				return
+			}
+			if z.Cache == nil {
+				z.Cache = make(map[string]dataUsageEntryV7, zb0002)
+			} else if len(z.Cache) > 0 {
+				for key := range z.Cache {
+					delete(z.Cache, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 dataUsageEntryV7
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache")
+					return
+				}
+				bts, err = za0002.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Cache", za0001)
+					return
+				}
+				z.Cache[za0001] = za0002
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageCacheV7) Msgsize() (s int) {
+	s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
+	if z.Cache != nil {
+		for za0001, za0002 := range z.Cache {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
+		}
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -2684,6 +2827,297 @@ func (z *dataUsageEntryV6) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *dataUsageEntryV7) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ch":
+			err = z.Children.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "Children")
+				return
+			}
+		case "sz":
+			z.Size, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "os":
+			z.Objects, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "Objects")
+				return
+			}
+		case "vs":
+			z.Versions, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "dms":
+			z.DeleteMarkers, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "DeleteMarkers")
+				return
+			}
+		case "szs":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjSizes")
+				return
+			}
+			if zb0002 != uint32(dataUsageBucketLenV1) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0002}
+				return
+			}
+			for za0001 := range z.ObjSizes {
+				z.ObjSizes[za0001], err = dc.ReadUint64()
+				if err != nil {
+					err = msgp.WrapError(err, "ObjSizes", za0001)
+					return
+				}
+			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], err = dc.ReadUint64()
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
+		case "rs":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "ReplicationStats")
+					return
+				}
+				z.ReplicationStats = nil
+			} else {
+				if z.ReplicationStats == nil {
+					z.ReplicationStats = new(replicationAllStats)
+				}
+				err = z.ReplicationStats.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "ReplicationStats")
+					return
+				}
+			}
+		case "ats":
+			if dc.IsNil() {
+				err = dc.ReadNil()
+				if err != nil {
+					err = msgp.WrapError(err, "AllTierStats")
+					return
+				}
+				z.AllTierStats = nil
+			} else {
+				if z.AllTierStats == nil {
+					z.AllTierStats = new(allTierStats)
+				}
+				err = z.AllTierStats.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "AllTierStats")
+					return
+				}
+			}
+		case "c":
+			z.Compacted, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "Compacted")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *dataUsageEntryV7) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ch":
+			bts, err = z.Children.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Children")
+				return
+			}
+		case "sz":
+			z.Size, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Size")
+				return
+			}
+		case "os":
+			z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Objects")
+				return
+			}
+		case "vs":
+			z.Versions, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "dms":
+			z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "DeleteMarkers")
+				return
+			}
+		case "szs":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjSizes")
+				return
+			}
+			if zb0002 != uint32(dataUsageBucketLenV1) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0002}
+				return
+			}
+			for za0001 := range z.ObjSizes {
+				z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ObjSizes", za0001)
+					return
+				}
+			}
+		case "vh":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ObjVersions")
+				return
+			}
+			if zb0003 != uint32(dataUsageVersionLen) {
+				err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
+				return
+			}
+			for za0002 := range z.ObjVersions {
+				z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ObjVersions", za0002)
+					return
+				}
+			}
+		case "rs":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.ReplicationStats = nil
+			} else {
+				if z.ReplicationStats == nil {
+					z.ReplicationStats = new(replicationAllStats)
+				}
+				bts, err = z.ReplicationStats.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "ReplicationStats")
+					return
+				}
+			}
+		case "ats":
+			if msgp.IsNil(bts) {
+				bts, err = msgp.ReadNilBytes(bts)
+				if err != nil {
+					return
+				}
+				z.AllTierStats = nil
+			} else {
+				if z.AllTierStats == nil {
+					z.AllTierStats = new(allTierStats)
+				}
+				bts, err = z.AllTierStats.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "AllTierStats")
+					return
+				}
+			}
+		case "c":
+			z.Compacted, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Compacted")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *dataUsageEntryV7) Msgsize() (s int) {
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLenV1 * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
+	if z.ReplicationStats == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.ReplicationStats.Msgsize()
+	}
+	s += 4
+	if z.AllTierStats == nil {
+		s += msgp.NilSize
+	} else {
+		s += z.AllTierStats.Msgsize()
+	}
+	s += 2 + msgp.BoolSize
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
 	{
@@ -3705,6 +4139,84 @@ func (z *sizeHistogram) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *sizeHistogramV1) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0001 uint32
+	zb0001, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLenV1) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], err = dc.ReadUint64()
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *sizeHistogramV1) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(dataUsageBucketLenV1))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for za0001 := range z {
+		err = en.WriteUint64(z[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *sizeHistogramV1) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLenV1))
+	for za0001 := range z {
+		o = msgp.AppendUint64(o, z[za0001])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *sizeHistogramV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if zb0001 != uint32(dataUsageBucketLenV1) {
+		err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0001}
+		return
+	}
+	for za0001 := range z {
+		z[za0001], bts, err = msgp.ReadUint64Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, za0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *sizeHistogramV1) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize + (dataUsageBucketLenV1 * (msgp.Uint64Size))
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *tierStats) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/data-usage-cache_gen_test.go

@@ -1084,6 +1084,119 @@ func BenchmarkDecodesizeHistogram(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalsizeHistogramV1(t *testing.T) {
+	v := sizeHistogramV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgsizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgsizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalsizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodesizeHistogramV1(t *testing.T) {
+	v := sizeHistogramV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodesizeHistogramV1 Msgsize() is inaccurate")
+	}
+
+	vn := sizeHistogramV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodesizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodesizeHistogramV1(b *testing.B) {
+	v := sizeHistogramV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshaltierStats(t *testing.T) {
 	v := tierStats{}
 	bts, err := v.MarshalMsg(nil)

cmd/data-usage-cache_test.go

@@ -0,0 +1,98 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/dustin/go-humanize"
+)
+
+func TestSizeHistogramToMap(t *testing.T) {
+	tests := []struct {
+		sizes []int64
+		want  map[string]uint64
+	}{
+		{
+			sizes: []int64{100, 1000, 72_000, 100_000},
+			want: map[string]uint64{
+				"LESS_THAN_1024_B":         2,
+				"BETWEEN_64_KB_AND_256_KB": 2,
+				"BETWEEN_1024B_AND_1_MB":   2,
+			},
+		},
+		{
+			sizes: []int64{100, 1000, 2000, 100_000, 13 * humanize.MiByte},
+			want: map[string]uint64{
+				"LESS_THAN_1024_B":         2,
+				"BETWEEN_1024_B_AND_64_KB": 1,
+				"BETWEEN_64_KB_AND_256_KB": 1,
+				"BETWEEN_1024B_AND_1_MB":   2,
+				"BETWEEN_10_MB_AND_64_MB":  1,
+			},
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("Test-%d", i), func(t *testing.T) {
+			var h sizeHistogram
+			for _, sz := range test.sizes {
+				h.add(sz)
+			}
+			got := h.toMap()
+			exp := test.want
+			// what is in exp is in got
+			for k := range exp {
+				if exp[k] != got[k] {
+					t.Fatalf("interval %s: Expected %d values but got %d values\n", k, exp[k], got[k])
+				}
+			}
+			// what is absent in exp is absent in got too
+			for k := range got {
+				if _, ok := exp[k]; !ok && got[k] > 0 {
+					t.Fatalf("Unexpected interval: %s has value %d\n", k, got[k])
+				}
+			}
+		})
+	}
+}
+
+func TestMigrateSizeHistogramFromV1(t *testing.T) {
+	tests := []struct {
+		v    sizeHistogramV1
+		want sizeHistogram
+	}{
+		{
+			v:    sizeHistogramV1{0: 10, 1: 20, 2: 3},
+			want: sizeHistogram{0: 10, 5: 20, 6: 3},
+		},
+		{
+			v:    sizeHistogramV1{0: 10, 1: 20, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7},
+			want: sizeHistogram{0: 10, 5: 20, 6: 3, 7: 4, 8: 5, 9: 6, 10: 7},
+		},
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) {
+			var got sizeHistogram
+			got.mergeV1(test.v)
+			if got != test.want {
+				t.Fatalf("Expected %v but got %v", test.want, got)
+			}
+		})
+	}
+}

cmd/data-usage_test.go

@@ -82,7 +82,7 @@ func TestDataUsageUpdate(t *testing.T) {
 			size:    1322310,
 			flatten: true,
 			objs:    8,
-			oSizes:  sizeHistogram{0: 2, 1: 6},
+			oSizes:  sizeHistogram{0: 2, 1: 3, 2: 2, 4: 1},
 		},
 		{
 			path:   "/",
@@ -94,7 +94,7 @@ func TestDataUsageUpdate(t *testing.T) {
 			path:   "/dir1",
 			size:   1302010,
 			objs:   5,
-			oSizes: sizeHistogram{0: 1, 1: 4},
+			oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1},
 		},
 		{
 			path:  "/dir1/dira",
@@ -167,7 +167,6 @@ func TestDataUsageUpdate(t *testing.T) {
 			size: 200,
 		},
 	}
-
 	createUsageTestFiles(t, base, bucket, files)
 	err = os.RemoveAll(filepath.Join(base, bucket, "dir1/dira/dirasub/dcfile"))
 	if err != nil {
@@ -194,14 +193,14 @@ func TestDataUsageUpdate(t *testing.T) {
 			size:    363515,
 			flatten: true,
 			objs:    14,
-			oSizes:  sizeHistogram{0: 7, 1: 7},
+			oSizes:  sizeHistogram{0: 7, 1: 5, 2: 2},
 		},
 		{
 			path:    "/dir1",
 			size:    342210,
 			objs:    7,
 			flatten: false,
-			oSizes:  sizeHistogram{0: 2, 1: 5},
+			oSizes:  sizeHistogram{0: 2, 1: 3, 2: 2},
 		},
 		{
 			path:   "/newfolder",
@@ -308,7 +307,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			path:   "flat",
 			size:   1322310 + expectSize,
 			objs:   8 + expectSize,
-			oSizes: sizeHistogram{0: 2 + expectSize, 1: 6},
+			oSizes: sizeHistogram{0: 2 + expectSize, 1: 3, 2: 2, 4: 1},
 		},
 		{
 			path:   "bucket/",
@@ -321,7 +320,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			path:   "bucket/dir1",
 			size:   1302010,
 			objs:   5,
-			oSizes: sizeHistogram{0: 1, 1: 4},
+			oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1},
 		},
 		{
 			// Gets compacted at this level...
@@ -436,13 +435,13 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 			path:   "flat",
 			size:   363515 + expectSize,
 			objs:   14 + expectSize,
-			oSizes: sizeHistogram{0: 7 + expectSize, 1: 7},
+			oSizes: sizeHistogram{0: 7 + expectSize, 1: 5, 2: 2},
 		},
 		{
 			path:   "bucket/dir1",
 			size:   342210,
 			objs:   7,
-			oSizes: sizeHistogram{0: 2, 1: 5},
+			oSizes: sizeHistogram{0: 2, 1: 3, 2: 2},
 		},
 		{
 			path:   "bucket/",

cmd/endpoint.go

@@ -103,7 +103,7 @@ func (endpoint Endpoint) GridHost() string {
 
 // UpdateIsLocal - resolves the host and updates if it is local or not.
 func (endpoint *Endpoint) UpdateIsLocal() (err error) {
-	if !endpoint.IsLocal {
+	if endpoint.Host != "" {
 		endpoint.IsLocal, err = isLocalHost(endpoint.Hostname(), endpoint.Port(), globalMinioPort)
 		if err != nil {
 			return err
@@ -522,6 +522,16 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 					continue
 				}
 
+				if endpoints[i].Host == "" {
+					resolvedList[i] = true
+					endpoints[i].IsLocal = true
+					epsResolved++
+					if !foundLocal {
+						foundLocal = true
+					}
+					continue
+				}
+
 				// Log the message to console about the host resolving
 				reqInfo := (&logger.ReqInfo{}).AppendTags(
 					"host",
@@ -705,6 +715,17 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 						continue
 					}
 
+					if endpoint.Host == "" {
+						if !foundLocal {
+							foundLocal = true
+						}
+						endpoint.IsLocal = true
+						endpoints[j] = endpoint
+						epsResolved++
+						resolvedList[endpoint] = true
+						continue
+					}
+
 					// Log the message to console about the host resolving
 					reqInfo := (&logger.ReqInfo{}).AppendTags(
 						"host",
@@ -859,6 +880,8 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 		return poolEndpoints, setupType, nil
 	}
 
+	uniqueArgs := set.NewStringSet()
+
 	for poolIdx, pool := range poolsLayout {
 		var endpoints Endpoints
 		for setIdx, setLayout := range pool.layout {
@@ -889,26 +912,10 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 		poolEndpoints[poolIdx] = endpoints
 	}
 
-	for _, endpoints := range poolEndpoints {
-		// Return Erasure setup when all endpoints are path style.
-		if endpoints[0].Type() == PathEndpointType {
-			setupType = ErasureSetupType
-		}
-		if endpoints[0].Type() == URLEndpointType && setupType != DistErasureSetupType {
-			setupType = DistErasureSetupType
-		}
-	}
-
-	if setupType == ErasureSetupType {
-		return poolEndpoints, setupType, nil
-	}
-
 	if err := poolEndpoints.UpdateIsLocal(); err != nil {
 		return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}
 
-	uniqueArgs := set.NewStringSet()
-
 	for i, endpoints := range poolEndpoints {
 		// Here all endpoints are URL style.
 		endpointPathSet := set.NewStringSet()
@@ -918,7 +925,7 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 
 		for _, endpoint := range endpoints {
 			endpointPathSet.Add(endpoint.Path)
-			if endpoint.IsLocal {
+			if endpoint.IsLocal && endpoint.Host != "" {
 				localServerHostSet.Add(endpoint.Hostname())
 
 				_, port, err := net.SplitHostPort(endpoint.Host)
@@ -934,9 +941,8 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 		orchestrated := IsKubernetes() || IsDocker()
 		reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
 		// If not orchestrated
-		if !orchestrated &&
-			// and not setup in reverse proxy
-			!reverseProxy {
+		// and not setup in reverse proxy
+		if !orchestrated && !reverseProxy {
 			// Check whether same path is not used in endpoints of a host on different port.
 			// Only verify this on baremetal setups, DNS is not available in orchestrated
 			// environments so we can't do much here.
@@ -944,14 +950,18 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 			hostIPCache := make(map[string]set.StringSet)
 			for _, endpoint := range endpoints {
 				host := endpoint.Hostname()
-				hostIPSet, ok := hostIPCache[host]
-				if !ok {
-					var err error
-					hostIPSet, err = getHostIP(host)
-					if err != nil {
-						return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+				var hostIPSet set.StringSet
+				if host != "" {
+					var ok bool
+					hostIPSet, ok = hostIPCache[host]
+					if !ok {
+						var err error
+						hostIPSet, err = getHostIP(host)
+						if err != nil {
+							return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+						}
+						hostIPCache[host] = hostIPSet
 					}
-					hostIPCache[host] = hostIPSet
 				}
 				if IPSet, ok := pathIPMap[endpoint.Path]; ok {
 					if !IPSet.Intersection(hostIPSet).IsEmpty() {
@@ -982,12 +992,14 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 
 		// Add missing port in all endpoints.
 		for i := range endpoints {
-			_, port, err := net.SplitHostPort(endpoints[i].Host)
-			if err != nil {
-				endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
-			} else if endpoints[i].IsLocal && serverAddrPort != port {
-				// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
-				endpoints[i].IsLocal = false
+			if endpoints[i].Host != "" {
+				_, port, err := net.SplitHostPort(endpoints[i].Host)
+				if err != nil {
+					endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
+				} else if endpoints[i].IsLocal && serverAddrPort != port {
+					// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
+					endpoints[i].IsLocal = false
+				}
 			}
 		}
 
@@ -1006,10 +1018,12 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 			// This means it is DistErasure setup.
 		}
 
-		poolEndpoints[i] = endpoints
-
 		for _, endpoint := range endpoints {
-			uniqueArgs.Add(endpoint.Host)
+			if endpoint.Host != "" {
+				uniqueArgs.Add(endpoint.Host)
+			} else {
+				uniqueArgs.Add(net.JoinHostPort("localhost", serverAddrPort))
+			}
 		}
 
 		poolEndpoints[i] = endpoints
@@ -1020,13 +1034,21 @@ func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]E
 		updateDomainIPs(uniqueArgs)
 	}
 
+	erasureType := len(uniqueArgs.ToSlice()) == 1
+
 	for _, endpoints := range poolEndpoints {
 		// Return Erasure setup when all endpoints are path style.
 		if endpoints[0].Type() == PathEndpointType {
 			setupType = ErasureSetupType
+			break
 		}
-		if endpoints[0].Type() == URLEndpointType && setupType != DistErasureSetupType {
-			setupType = DistErasureSetupType
+		if endpoints[0].Type() == URLEndpointType {
+			if erasureType {
+				setupType = ErasureSetupType
+			} else {
+				setupType = DistErasureSetupType
+			}
+			break
 		}
 	}
 

cmd/endpoint_test.go

@@ -258,7 +258,7 @@ func TestCreateEndpoints(t *testing.T) {
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d2"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d3"}, IsLocal: true},
 			Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d4"}, IsLocal: true},
-		}, DistErasureSetupType, nil},
+		}, ErasureSetupType, nil},
 		// DistErasure Setup with URLEndpointType having mixed naming to local host.
 		{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")},
 

cmd/erasure-sets.go

@@ -128,9 +128,11 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 		if errors.Is(err, errUnformattedDisk) {
 			info, derr := disk.DiskInfo(context.TODO(), false)
 			if derr != nil && info.RootDisk {
+				disk.Close()
 				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
 			}
 		}
+		disk.Close()
 		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
@@ -413,6 +415,7 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 						globalLocalDrivesMu.RUnlock()
 						continue
 					}
+					disk.Close()
 					disk = ldisk
 					globalLocalDrivesMu.RUnlock()
 				}
@@ -1053,6 +1056,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 			// defective drive we should not heal a path on the root disk.
 			logger.LogIf(GlobalContext, fmt.Errorf("Drive `%s` is part of root drive, will not be used", storageDisks[i]))
+			storageDisks[i].Close()
 			storageDisks[i] = nil
 		}
 	}
@@ -1062,7 +1066,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
 	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, storageOpts{
 		cleanUp:     true,
-		healthCheck: true,
+		healthCheck: false,
 	})
 
 	defer func(storageDisks []StorageAPI) {
@@ -1132,6 +1136,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			}
 			if err := saveFormatErasure(storageDisks[index], format, formatOpID); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
+				storageDisks[index].Close()
 				tmpNewFormats[index] = nil // this disk failed to write new format
 			}
 		}
@@ -1154,17 +1159,27 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			}
 
 			if disk := storageDisks[index]; disk != nil {
-				disk.SetDiskLoc(s.poolIndex, m, n)
+				if disk.IsLocal() {
+					disk.SetDiskLoc(s.poolIndex, m, n)
 
-				if disk.IsLocal() && driveQuorum {
-					commonWrites, commonDeletes := calcCommonWritesDeletes(currentDisksInfo[m], (s.setDriveCount+1)/2)
 					xldisk, ok := disk.(*xlStorageDiskIDCheck)
 					if ok {
-						xldisk.totalWrites.Add(commonWrites)
-						xldisk.totalDeletes.Add(commonDeletes)
-						xldisk.storage.setWriteAttribute(commonWrites)
-						xldisk.storage.setDeleteAttribute(commonDeletes)
+						if driveQuorum {
+							commonWrites, commonDeletes := calcCommonWritesDeletes(currentDisksInfo[m], (s.setDriveCount+1)/2)
+							xldisk.totalWrites.Store(commonWrites)
+							xldisk.totalDeletes.Store(commonDeletes)
+							xldisk.storage.setWriteAttribute(commonWrites)
+							xldisk.storage.setDeleteAttribute(commonDeletes)
+						}
+						go xldisk.monitorDiskWritable(xldisk.diskCtx)
 					}
+				} else {
+					disk.Close() // Close the remote storage client, re-initialize with healthchecks.
+					disk, err = newStorageRESTClient(disk.Endpoint(), true, globalGrid.Load())
+					if err != nil {
+						continue
+					}
+					disk.SetDiskLoc(s.poolIndex, m, n)
 				}
 
 				s.erasureDisks[m][n] = disk

cmd/erasure.go

@@ -91,7 +91,7 @@ func diskErrToDriveState(err error) (state string) {
 	switch {
 	case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded):
 		state = madmin.DriveStateOffline
-	case errors.Is(err, errCorruptedFormat):
+	case errors.Is(err, errCorruptedFormat) || errors.Is(err, errCorruptedBackend):
 		state = madmin.DriveStateCorrupt
 	case errors.Is(err, errUnformattedDisk):
 		state = madmin.DriveStateUnformatted

cmd/object-api-datatypes.go

@@ -54,16 +54,35 @@ type objectHistogramInterval struct {
 }
 
 const (
+	// dataUsageBucketLenV1 must be length of ObjectsHistogramIntervalsV1
+	dataUsageBucketLenV1 = 7
 	// dataUsageBucketLen must be length of ObjectsHistogramIntervals
-	dataUsageBucketLen  = 7
+	dataUsageBucketLen  = 11
 	dataUsageVersionLen = 7
 )
 
+// ObjectsHistogramIntervalsV1 is the list of all intervals
+// of object sizes to be included in objects histogram(V1).
+var ObjectsHistogramIntervalsV1 = [dataUsageBucketLenV1]objectHistogramInterval{
+	{"LESS_THAN_1024_B", 0, humanize.KiByte - 1},
+	{"BETWEEN_1024B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
+	{"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1},
+	{"BETWEEN_10_MB_AND_64_MB", humanize.MiByte * 10, humanize.MiByte*64 - 1},
+	{"BETWEEN_64_MB_AND_128_MB", humanize.MiByte * 64, humanize.MiByte*128 - 1},
+	{"BETWEEN_128_MB_AND_512_MB", humanize.MiByte * 128, humanize.MiByte*512 - 1},
+	{"GREATER_THAN_512_MB", humanize.MiByte * 512, math.MaxInt64},
+}
+
 // ObjectsHistogramIntervals is the list of all intervals
 // of object sizes to be included in objects histogram.
+// Note: this histogram expands 1024B-1MB to incl. 1024B-64KB, 64KB-256KB, 256KB-512KB and 512KB-1MiB
 var ObjectsHistogramIntervals = [dataUsageBucketLen]objectHistogramInterval{
 	{"LESS_THAN_1024_B", 0, humanize.KiByte - 1},
-	{"BETWEEN_1024_B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
+	{"BETWEEN_1024_B_AND_64_KB", humanize.KiByte, 64*humanize.KiByte - 1},         // not exported, for support use only
+	{"BETWEEN_64_KB_AND_256_KB", 64 * humanize.KiByte, 256*humanize.KiByte - 1},   // not exported, for support use only
+	{"BETWEEN_256_KB_AND_512_KB", 256 * humanize.KiByte, 512*humanize.KiByte - 1}, // not exported, for support use only
+	{"BETWEEN_512_KB_AND_1_MB", 512 * humanize.KiByte, humanize.MiByte - 1},       // not exported, for support use only
+	{"BETWEEN_1024B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1},
 	{"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1},
 	{"BETWEEN_10_MB_AND_64_MB", humanize.MiByte * 10, humanize.MiByte*64 - 1},
 	{"BETWEEN_64_MB_AND_128_MB", humanize.MiByte * 64, humanize.MiByte*128 - 1},

cmd/object-handlers.go

@@ -3304,8 +3304,25 @@ func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *h
 
 	ot, err := objAPI.GetObjectTags(ctx, bucket, object, opts)
 	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
+		// if object/version is not found locally, but exists on peer site - proxy
+		// the tagging request to peer site. The response to client will
+		// return tags from peer site.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			proxytgts := getProxyTargets(ctx, bucket, object, opts)
+			if !proxytgts.Empty() {
+				// proxy to replication target if site replication is in place.
+				tags, gerr := proxyGetTaggingToRepTarget(ctx, bucket, object, opts, proxytgts)
+				if gerr.Err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, gerr.Err), r.URL)
+					return
+				} // overlay tags from peer site.
+				ot = tags
+				w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} // indicate that the request was proxied.
+			}
+		} else {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
 	}
 
 	// Set this such that authorization policies can be applied on the object tags.
@@ -3385,6 +3402,33 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 
 	objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
+		// if object is not found locally, but exists on peer site - proxy
+		// the tagging request to peer site. The response to client will
+		// be 200 with extra header indicating that the request was proxied.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			proxytgts := getProxyTargets(ctx, bucket, object, opts)
+			if !proxytgts.Empty() {
+				// proxy to replication target if site replication is in place.
+				perr := proxyTaggingToRepTarget(ctx, bucket, object, tags, opts, proxytgts)
+				if perr.Err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
+					return
+				}
+				w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"}
+				writeSuccessResponseHeadersOnly(w)
+				// when tagging is proxied, the object version is not available to return
+				// as header in the response, or ObjectInfo in the notification event.
+				sendEvent(eventArgs{
+					EventName:    event.ObjectCreatedPutTagging,
+					BucketName:   bucket,
+					ReqParams:    extractReqParams(r),
+					RespElements: extractRespElements(w),
+					UserAgent:    r.UserAgent(),
+					Host:         handlers.GetSourceIP(r),
+				})
+				return
+			}
+		}
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -3453,6 +3497,34 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 
 	oi, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
+		// if object is not found locally, but exists on peer site - proxy
+		// the tagging request to peer site. The response to client will
+		// be 200 OK with extra header indicating that the request was proxied.
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			proxytgts := getProxyTargets(ctx, bucket, object, opts)
+			if !proxytgts.Empty() {
+				// proxy to replication target if active-active replication is in place.
+				perr := proxyTaggingToRepTarget(ctx, bucket, object, nil, opts, proxytgts)
+				if perr.Err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL)
+					return
+				}
+				// when delete tagging is proxied, the object version/tags are not available to return
+				// as header in the response, nor ObjectInfo in the notification event.
+				w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"}
+				writeSuccessNoContent(w)
+				sendEvent(eventArgs{
+					EventName:    event.ObjectCreatedDeleteTagging,
+					BucketName:   bucket,
+					Object:       oi,
+					ReqParams:    extractReqParams(r),
+					RespElements: extractRespElements(w),
+					UserAgent:    r.UserAgent(),
+					Host:         handlers.GetSourceIP(r),
+				})
+				return
+			}
+		}
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}

cmd/storage-errors.go

@@ -30,8 +30,11 @@ var errMaxVersionsExceeded = StorageErr("maximum versions exceeded, please delet
 // errUnexpected - unexpected error, requires manual intervention.
 var errUnexpected = StorageErr("unexpected error, please report this issue at https://github.com/minio/minio/issues")
 
-// errCorruptedFormat - corrupted backend format.
-var errCorruptedFormat = StorageErr("corrupted backend format, specified drive mount has unexpected previous content")
+// errCorruptedFormat - corrupted format.
+var errCorruptedFormat = StorageErr("corrupted format")
+
+// errCorruptedBackend - corrupted backend.
+var errCorruptedBackend = StorageErr("corrupted backend")
 
 // errUnformattedDisk - unformatted disk found.
 var errUnformattedDisk = StorageErr("unformatted drive found")

cmd/storage-rest-client.go

@@ -115,6 +115,8 @@ func toStorageErr(err error) error {
 		return errVolumeAccessDenied
 	case errCorruptedFormat.Error():
 		return errCorruptedFormat
+	case errCorruptedBackend.Error():
+		return errCorruptedBackend
 	case errUnformattedDisk.Error():
 		return errUnformattedDisk
 	case errInvalidAccessKeyID.Error():

cmd/xl-storage-disk-id-check.go

@@ -228,8 +228,8 @@ func newXLStorageDiskIDCheck(storage *xlStorage, healthCheck bool) *xlStorageDis
 	}
 
 	if driveQuorum {
-		xl.totalWrites.Add(xl.storage.getWriteAttribute())
-		xl.totalDeletes.Add(xl.storage.getDeleteAttribute())
+		xl.totalWrites.Store(xl.storage.getWriteAttribute())
+		xl.totalDeletes.Store(xl.storage.getDeleteAttribute())
 	}
 
 	xl.diskCtx, xl.cancel = context.WithCancel(context.TODO())
@@ -1032,37 +1032,50 @@ func (p *xlStorageDiskIDCheck) checkHealth(ctx context.Context) (err error) {
 	if t > maxTimeSinceLastSuccess {
 		if atomic.CompareAndSwapInt32(&p.health.status, diskHealthOK, diskHealthFaulty) {
 			logger.LogAlwaysIf(ctx, fmt.Errorf("node(%s): taking drive %s offline, time since last response %v", globalLocalNodeName, p.storage.String(), t.Round(time.Millisecond)))
-			go p.monitorDiskStatus(t)
+			go p.monitorDiskStatus(0, mustGetUUID())
 		}
 		return errFaultyDisk
 	}
 	return nil
 }
 
+// Make sure we do not write O_DIRECT aligned I/O because WrIteAll() ends
+// up using O_DIRECT codepath which internally utilizes p.health.tokens
+// we need to avoid using incoming I/O tokens as part of the healthcheck
+// monitoring I/O.
+var toWrite = []byte{2048: 42}
+
 // monitorDiskStatus should be called once when a drive has been marked offline.
 // Once the disk has been deemed ok, it will return to online status.
-func (p *xlStorageDiskIDCheck) monitorDiskStatus(spent time.Duration) {
+func (p *xlStorageDiskIDCheck) monitorDiskStatus(spent time.Duration, fn string) {
 	t := time.NewTicker(5 * time.Second)
 	defer t.Stop()
 
-	fn := mustGetUUID()
 	for range t.C {
+		if contextCanceled(p.diskCtx) {
+			return
+		}
+
 		if len(p.health.tokens) == 0 {
 			// Queue is still full, no need to check.
 			continue
 		}
-		err := p.storage.WriteAll(context.Background(), minioMetaTmpBucket, fn, []byte{10000: 42})
+
+		err := p.storage.WriteAll(context.Background(), minioMetaTmpBucket, fn, toWrite)
 		if err != nil {
 			continue
 		}
+
 		b, err := p.storage.ReadAll(context.Background(), minioMetaTmpBucket, fn)
-		if err != nil || len(b) != 10001 {
+		if err != nil || len(b) != len(toWrite) {
 			continue
 		}
+
 		err = p.storage.Delete(context.Background(), minioMetaTmpBucket, fn, DeleteOptions{
 			Recursive: false,
 			Immediate: false,
 		})
+
 		if err == nil {
 			t := time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess))
 			if spent > 0 {
@@ -1108,8 +1121,6 @@ func (p *xlStorageDiskIDCheck) monitorDiskWritable(ctx context.Context) {
 	defer t.Stop()
 	fn := mustGetUUID()
 
-	// Be just above directio size.
-	toWrite := []byte{xioutil.DirectioAlignSize + 1: 42}
 	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	monitor := func() bool {
@@ -1129,7 +1140,7 @@ func (p *xlStorageDiskIDCheck) monitorDiskWritable(ctx context.Context) {
 		goOffline := func(err error, spent time.Duration) {
 			if atomic.CompareAndSwapInt32(&p.health.status, diskHealthOK, diskHealthFaulty) {
 				logger.LogAlwaysIf(ctx, fmt.Errorf("node(%s): taking drive %s offline: %v", globalLocalNodeName, p.storage.String(), err))
-				go p.monitorDiskStatus(spent)
+				go p.monitorDiskStatus(spent, fn)
 			}
 		}
 

cmd/xl-storage.go

@@ -715,11 +715,7 @@ func (s *xlStorage) DiskInfo(_ context.Context, _ bool) (info DiskInfo, err erro
 	s.diskInfoCache.Once.Do(func() {
 		s.diskInfoCache.TTL = time.Second
 		s.diskInfoCache.Update = func() (interface{}, error) {
-			dcinfo := DiskInfo{
-				RootDisk:  s.rootDisk,
-				MountPath: s.drivePath,
-				Endpoint:  s.endpoint.String(),
-			}
+			dcinfo := DiskInfo{}
 			di, err := getDiskInfo(s.drivePath)
 			if err != nil {
 				return dcinfo, err
@@ -748,6 +744,10 @@ func (s *xlStorage) DiskInfo(_ context.Context, _ bool) (info DiskInfo, err erro
 	if v != nil {
 		info = v.(DiskInfo)
 	}
+
+	info.RootDisk = s.rootDisk
+	info.MountPath = s.drivePath
+	info.Endpoint = s.endpoint.String()
 	info.Scanning = atomic.LoadInt32(&s.scanning) == 1
 	return info, err
 }
@@ -779,12 +779,12 @@ func (s *xlStorage) checkFormatJSON() (os.FileInfo, error) {
 				return nil, errDiskAccessDenied
 			}
 			logger.LogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors
-			return nil, errCorruptedFormat
+			return nil, errCorruptedBackend
 		} else if osIsPermission(err) {
 			return nil, errDiskAccessDenied
 		}
 		logger.LogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors
-		return nil, errCorruptedFormat
+		return nil, errCorruptedBackend
 	}
 	return fi, nil
 }
@@ -830,12 +830,12 @@ func (s *xlStorage) GetDiskID() (string, error) {
 				return "", errDiskAccessDenied
 			}
 			logger.LogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors
-			return "", errCorruptedFormat
+			return "", errCorruptedBackend
 		} else if osIsPermission(err) {
 			return "", errDiskAccessDenied
 		}
 		logger.LogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors
-		return "", errCorruptedFormat
+		return "", errCorruptedBackend
 	}
 
 	format := &formatErasureV3{}

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2024-01-05T22-17-24Z
+  image: quay.io/minio/minio:RELEASE.2024-01-11T07-46-16Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 5.0.14
-appVersion: RELEASE.2023-09-30T07-02-29Z
+version: 5.0.15
+appVersion: RELEASE.2024-01-11T07-46-16Z
 keywords:
   - minio
   - storage

helm/minio/templates/_helper_create_bucket.txt

@@ -1,5 +1,5 @@
 #!/bin/sh
-set -e ; # Have script exit in the event of a failed command.
+set -e # Have script exit in the event of a failed command.
 
 {{- if .Values.configPathmc }}
 MC_CONFIG_DIR="{{ .Values.configPathmc }}"
@@ -11,99 +11,98 @@ MC="/usr/bin/mc --insecure"
 # connectToMinio
 # Use a check-sleep-check loop to wait for MinIO service to be available
 connectToMinio() {
-  SCHEME=$1
-  ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts
-  set -e ; # fail if we can't read the keys.
-  ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ;
-  set +e ; # The connections to minio are allowed to fail.
-  echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ;
-  MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ;
-  $MC_COMMAND ;
-  STATUS=$? ;
-  until [ $STATUS = 0 ]
-  do
-    ATTEMPTS=`expr $ATTEMPTS + 1` ;
-    echo \"Failed attempts: $ATTEMPTS\" ;
-    if [ $ATTEMPTS -gt $LIMIT ]; then
-      exit 1 ;
-    fi ;
-    sleep 2 ; # 1 second intervals between attempts
-    $MC_COMMAND ;
-    STATUS=$? ;
-  done ;
-  set -e ; # reset `e` as active
-  return 0
+	SCHEME=$1
+	ATTEMPTS=0
+	LIMIT=29 # Allow 30 attempts
+	set -e   # fail if we can't read the keys.
+	ACCESS=$(cat /config/rootUser)
+	SECRET=$(cat /config/rootPassword)
+	set +e # The connections to minio are allowed to fail.
+	echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT"
+	MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET"
+	$MC_COMMAND
+	STATUS=$?
+	until [ $STATUS = 0 ]; do
+		ATTEMPTS=$(expr $ATTEMPTS + 1)
+		echo \"Failed attempts: $ATTEMPTS\"
+		if [ $ATTEMPTS -gt $LIMIT ]; then
+			exit 1
+		fi
+		sleep 2 # 1 second intervals between attempts
+		$MC_COMMAND
+		STATUS=$?
+	done
+	set -e # reset `e` as active
+	return 0
 }
 
 # checkBucketExists ($bucket)
 # Check if the bucket exists, by using the exit code of `mc ls`
 checkBucketExists() {
-  BUCKET=$1
-  CMD=$(${MC} stat myminio/$BUCKET > /dev/null 2>&1)
-  return $?
+	BUCKET=$1
+	CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1)
+	return $?
 }
 
 # createBucket ($bucket, $policy, $purge)
 # Ensure bucket exists, purging if asked to
 createBucket() {
-  BUCKET=$1
-  POLICY=$2
-  PURGE=$3
-  VERSIONING=$4
-  OBJECTLOCKING=$5
+	BUCKET=$1
+	POLICY=$2
+	PURGE=$3
+	VERSIONING=$4
+	OBJECTLOCKING=$5
 
-  # Purge the bucket, if set & exists
-  # Since PURGE is user input, check explicitly for `true`
-  if [ $PURGE = true ]; then
-    if checkBucketExists $BUCKET ; then
-      echo "Purging bucket '$BUCKET'."
-      set +e ; # don't exit if this fails
-      ${MC} rm -r --force myminio/$BUCKET
-      set -e ; # reset `e` as active
-    else
-      echo "Bucket '$BUCKET' does not exist, skipping purge."
-    fi
-  fi
+	# Purge the bucket, if set & exists
+	# Since PURGE is user input, check explicitly for `true`
+	if [ $PURGE = true ]; then
+		if checkBucketExists $BUCKET; then
+			echo "Purging bucket '$BUCKET'."
+			set +e # don't exit if this fails
+			${MC} rm -r --force myminio/$BUCKET
+			set -e # reset `e` as active
+		else
+			echo "Bucket '$BUCKET' does not exist, skipping purge."
+		fi
+	fi
 
-# Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created)
-if ! checkBucketExists $BUCKET ; then
-    if [ ! -z $OBJECTLOCKING ] ; then
-      if [ $OBJECTLOCKING = true ] ; then
-          echo "Creating bucket with OBJECTLOCKING '$BUCKET'"
-          ${MC} mb --with-lock myminio/$BUCKET
-      elif [ $OBJECTLOCKING = false ] ; then
-            echo "Creating bucket '$BUCKET'"
-            ${MC} mb myminio/$BUCKET
-      fi
-  elif [ -z $OBJECTLOCKING ] ; then
-        echo "Creating bucket '$BUCKET'"
-        ${MC} mb myminio/$BUCKET
-  else
-    echo "Bucket '$BUCKET' already exists."  
-  fi
-  fi
+	# Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created)
+	if ! checkBucketExists $BUCKET; then
+		if [ ! -z $OBJECTLOCKING ]; then
+			if [ $OBJECTLOCKING = true ]; then
+				echo "Creating bucket with OBJECTLOCKING '$BUCKET'"
+				${MC} mb --with-lock myminio/$BUCKET
+			elif [ $OBJECTLOCKING = false ]; then
+				echo "Creating bucket '$BUCKET'"
+				${MC} mb myminio/$BUCKET
+			fi
+		elif [ -z $OBJECTLOCKING ]; then
+			echo "Creating bucket '$BUCKET'"
+			${MC} mb myminio/$BUCKET
+		else
+			echo "Bucket '$BUCKET' already exists."
+		fi
+	fi
 
+	# set versioning for bucket if objectlocking is disabled or not set
+	if [ $OBJECTLOCKING = false ]; then
+		if [ ! -z $VERSIONING ]; then
+			if [ $VERSIONING = true ]; then
+				echo "Enabling versioning for '$BUCKET'"
+				${MC} version enable myminio/$BUCKET
+			elif [ $VERSIONING = false ]; then
+				echo "Suspending versioning for '$BUCKET'"
+				${MC} version suspend myminio/$BUCKET
+			fi
+		fi
+	else
+		echo "Bucket '$BUCKET' versioning unchanged."
+	fi
 
-  # set versioning for bucket if objectlocking is disabled or not set
-  if [ $OBJECTLOCKING = false ] ; then
-  if [ ! -z $VERSIONING ] ; then
-    if [ $VERSIONING = true ] ; then
-        echo "Enabling versioning for '$BUCKET'"
-        ${MC} version enable myminio/$BUCKET
-    elif [ $VERSIONING = false ] ; then
-        echo "Suspending versioning for '$BUCKET'"
-        ${MC} version suspend myminio/$BUCKET
-    fi
-    fi
-  else
-      echo "Bucket '$BUCKET' versioning unchanged."
-  fi
-
-
-  # At this point, the bucket should exist, skip checking for existence
-  # Set policy on the bucket
-  echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
-  ${MC} anonymous set $POLICY myminio/$BUCKET
+	# At this point, the bucket should exist, skip checking for existence
+	# Set policy on the bucket
+	echo "Setting policy of bucket '$BUCKET' to '$POLICY'."
+	${MC} anonymous set $POLICY myminio/$BUCKET
 }
 
 # Try connecting to MinIO instance
@@ -118,6 +117,6 @@ connectToMinio $scheme
 {{ $global := . }}
 # Create the buckets
 {{- range .Values.buckets }}
-createBucket {{ tpl .name $global }} {{ .policy | default "none" | quote }} {{ .purge | default false }} {{ .versioning | default false }} {{ .objectlocking  | default false }}
+createBucket {{ tpl .name $global }} {{ .policy | default "none" | quote }} {{ .purge | default false }} {{ .versioning | default false }} {{ .objectlocking | default false }}
 {{- end }}
 {{- end }}

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2023-09-30T07-02-29Z
+  tag: RELEASE.2024-01-11T07-46-16Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,7 +25,7 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2023-09-29T16-41-22Z
+  tag: RELEASE.2024-01-11T05-49-32Z
   pullPolicy: IfNotPresent
 
 ## minio mode, i.e. standalone or distributed
@@ -411,7 +411,7 @@ buckets: []
   #   purge: false
   #   # set versioning for
   #   # bucket [true|false]
-  #   versioning: false
+  #   versioning: false # remove this key if you do not want versioning feature
   #   # set objectlocking for
   #   # bucket [true|false] NOTE: versioning is enabled by default if you use locking
   #   objectlocking: false

index.yaml

@@ -1,9 +1,31 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2024-01-11T07-46-16Z
+    created: "2024-01-12T10:18:32.881566033-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 3a2d8e03ffdd98501026aa7561633c91d9871647f4b01d77b75a2ad9b72ee618
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-5.0.15.tgz
+    version: 5.0.15
   - apiVersion: v1
     appVersion: RELEASE.2023-09-30T07-02-29Z
-    created: "2023-09-30T13:45:44.318107697-07:00"
+    created: "2024-01-12T10:18:32.880476811-08:00"
     description: Multi-Cloud Object Storage
     digest: 6c3656924fbad2cb17f810cd78f352f9b60626aaec64b837c96829095b215ad3
     home: https://min.io
@@ -25,7 +47,7 @@ entries:
     version: 5.0.14
   - apiVersion: v1
     appVersion: RELEASE.2023-07-07T07-13-57Z
-    created: "2023-09-30T13:45:44.317015357-07:00"
+    created: "2024-01-12T10:18:32.878777831-08:00"
     description: Multi-Cloud Object Storage
     digest: 3c18f7381efe6d86497f952e6d5f59003ee5a009c54778ddea1ee8d3c7bed9c8
     home: https://min.io
@@ -47,7 +69,7 @@ entries:
     version: 5.0.13
   - apiVersion: v1
     appVersion: RELEASE.2023-07-07T07-13-57Z
-    created: "2023-09-30T13:45:44.315355964-07:00"
+    created: "2024-01-12T10:18:32.877821671-08:00"
     description: Multi-Cloud Object Storage
     digest: 5318bc56c73a8f4539c3dd178f4d55c7f41bee4a25d7dc02ac6a5843eeee7976
     home: https://min.io
@@ -69,7 +91,7 @@ entries:
     version: 5.0.12
   - apiVersion: v1
     appVersion: RELEASE.2023-06-19T19-52-50Z
-    created: "2023-09-30T13:45:44.314320466-07:00"
+    created: "2024-01-12T10:18:32.876821354-08:00"
     description: Multi-Cloud Object Storage
     digest: cba44c8cddcda1fb5c082dce82004a39f53cc20677ab9698a6998f01efefd8db
     home: https://min.io
@@ -91,7 +113,7 @@ entries:
     version: 5.0.11
   - apiVersion: v1
     appVersion: RELEASE.2023-05-18T00-05-36Z
-    created: "2023-09-30T13:45:44.313340376-07:00"
+    created: "2024-01-12T10:18:32.875827968-08:00"
     description: Multi-Cloud Object Storage
     digest: a3d55b12f38a2049ddf3efe35b38b6dc4e59777452b72d18d5a82f3378deb9cd
     home: https://min.io
@@ -113,7 +135,7 @@ entries:
     version: 5.0.10
   - apiVersion: v1
     appVersion: RELEASE.2023-04-28T18-11-17Z
-    created: "2023-09-30T13:45:44.327034407-07:00"
+    created: "2024-01-12T10:18:32.889898236-08:00"
     description: Multi-Cloud Object Storage
     digest: cf98985e32675e4ce327304ea9ac61046a788b3d5190d6b501330f7803d41a11
     home: https://min.io
@@ -135,7 +157,7 @@ entries:
     version: 5.0.9
   - apiVersion: v1
     appVersion: RELEASE.2023-04-13T03-08-07Z
-    created: "2023-09-30T13:45:44.326090811-07:00"
+    created: "2024-01-12T10:18:32.888973878-08:00"
     description: Multi-Cloud Object Storage
     digest: 034d68f85799f6693836975797f85a91842cf2d003a6c4ff401bd4ea4c946af6
     home: https://min.io
@@ -157,7 +179,7 @@ entries:
     version: 5.0.8
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-09-30T13:45:44.32510969-07:00"
+    created: "2024-01-12T10:18:32.887985163-08:00"
     description: Multi-Cloud Object Storage
     digest: 3f935a310e1b5b873052629b66005c160356ca7b2bd394cb07b34dbaf9905e3f
     home: https://min.io
@@ -179,7 +201,7 @@ entries:
     version: 5.0.7
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-09-30T13:45:44.32366994-07:00"
+    created: "2024-01-12T10:18:32.887016642-08:00"
     description: Multi-Cloud Object Storage
     digest: 82ef858ce483c2d736444792986cb36bd0fb4fc90a80b97fe30d7b2f2034d24a
     home: https://min.io
@@ -201,7 +223,7 @@ entries:
     version: 5.0.6
   - apiVersion: v1
     appVersion: RELEASE.2023-01-31T02-24-19Z
-    created: "2023-09-30T13:45:44.322210657-07:00"
+    created: "2024-01-12T10:18:32.885381555-08:00"
     description: Multi-Cloud Object Storage
     digest: fefeea10e4e525e45f82fb80a03900d34605ec432dd92f56d94eaf4fb1b98c41
     home: https://min.io
@@ -223,7 +245,7 @@ entries:
     version: 5.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-09-30T13:45:44.321254746-07:00"
+    created: "2024-01-12T10:18:32.884404357-08:00"
     description: Multi-Cloud Object Storage
     digest: 6b305783c98b0b97ffab079ff4430094fd0ca6e98e82bb8153cb93033a1bf40f
     home: https://min.io
@@ -245,7 +267,7 @@ entries:
     version: 5.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-09-30T13:45:44.320169821-07:00"
+    created: "2024-01-12T10:18:32.88346955-08:00"
     description: Multi-Cloud Object Storage
     digest: bac89157c53b324aece263c294aa49f5c9b64f426b4b06c9bca3d72e77e244f2
     home: https://min.io
@@ -267,7 +289,7 @@ entries:
     version: 5.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-09-30T13:45:44.319086339-07:00"
+    created: "2024-01-12T10:18:32.882529564-08:00"
     description: Multi-Cloud Object Storage
     digest: 935ce4f09366231b11d414d626f887fa6fa6024dd30a42e81e810ca1438d5904
     home: https://min.io
@@ -289,7 +311,7 @@ entries:
     version: 5.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-11-11T03-44-20Z
-    created: "2023-09-30T13:45:44.312066393-07:00"
+    created: "2024-01-12T10:18:32.874842921-08:00"
     description: Multi-Cloud Object Storage
     digest: 3e952c5d737980b8ccdfb819021eafb4b4e8da226f764a1dc3de1ba63ceb1ffa
     home: https://min.io
@@ -311,7 +333,7 @@ entries:
     version: 5.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-09-30T13:45:44.311113045-07:00"
+    created: "2024-01-12T10:18:32.873801347-08:00"
     description: Multi-Cloud Object Storage
     digest: 6215c800d84fd4c40e4fb4142645fc1c6a039c251776a3cc8c11a24b9e3b59c7
     home: https://min.io
@@ -333,7 +355,7 @@ entries:
     version: 5.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-09-30T13:45:44.310137758-07:00"
+    created: "2024-01-12T10:18:32.871930814-08:00"
     description: Multi-Cloud Object Storage
     digest: 2d3d884490ea1127742f938bc9382844bae713caae08b3308f766f3c9000659a
     home: https://min.io
@@ -355,7 +377,7 @@ entries:
     version: 4.1.0
   - apiVersion: v1
     appVersion: RELEASE.2022-09-17T00-09-45Z
-    created: "2023-09-30T13:45:44.299926747-07:00"
+    created: "2024-01-12T10:18:32.863177419-08:00"
     description: Multi-Cloud Object Storage
     digest: 6f16f2dbfed91ab81a7fae60b6ea32f554365bd27bf5fda55b64a0fa264f4252
     home: https://min.io
@@ -377,7 +399,7 @@ entries:
     version: 4.0.15
   - apiVersion: v1
     appVersion: RELEASE.2022-09-01T23-53-36Z
-    created: "2023-09-30T13:45:44.298411641-07:00"
+    created: "2024-01-12T10:18:32.862228195-08:00"
     description: Multi-Cloud Object Storage
     digest: 35d89d8f49d53ea929466fb88ee26123431326033f1387e6b2d536a629c0a398
     home: https://min.io
@@ -399,7 +421,7 @@ entries:
     version: 4.0.14
   - apiVersion: v1
     appVersion: RELEASE.2022-08-22T23-53-06Z
-    created: "2023-09-30T13:45:44.29745713-07:00"
+    created: "2024-01-12T10:18:32.8612748-08:00"
     description: Multi-Cloud Object Storage
     digest: 5b86937ca88d9f6046141fdc2b1cc54760435ed92d289cd0a115fa7148781d4e
     home: https://min.io
@@ -421,7 +443,7 @@ entries:
     version: 4.0.13
   - apiVersion: v1
     appVersion: RELEASE.2022-08-13T21-54-44Z
-    created: "2023-09-30T13:45:44.296233683-07:00"
+    created: "2024-01-12T10:18:32.860151912-08:00"
     description: Multi-Cloud Object Storage
     digest: 2d9c227c0f46ea8bdef4d760c212156fd4c6623ddc5406779c569fe925527787
     home: https://min.io
@@ -443,7 +465,7 @@ entries:
     version: 4.0.12
   - apiVersion: v1
     appVersion: RELEASE.2022-08-05T23-27-09Z
-    created: "2023-09-30T13:45:44.295016135-07:00"
+    created: "2024-01-12T10:18:32.858318972-08:00"
     description: Multi-Cloud Object Storage
     digest: 6caaffcb636e040cd7e8bc4883a1674a673757f4781c32d53b5ec0f41fea3944
     home: https://min.io
@@ -465,7 +487,7 @@ entries:
     version: 4.0.11
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-09-30T13:45:44.29308946-07:00"
+    created: "2024-01-12T10:18:32.857387656-08:00"
     description: Multi-Cloud Object Storage
     digest: 841d87788fb094d6a7d8a91e91821fe1e847bc952e054c781fc93742d112e18a
     home: https://min.io
@@ -487,7 +509,7 @@ entries:
     version: 4.0.10
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-09-30T13:45:44.308504002-07:00"
+    created: "2024-01-12T10:18:32.87096148-08:00"
     description: Multi-Cloud Object Storage
     digest: 6f1a78382df3215deac07495a5e7de7009a1153b4cf6cb565630652a69aec4cf
     home: https://min.io
@@ -509,7 +531,7 @@ entries:
     version: 4.0.9
   - apiVersion: v1
     appVersion: RELEASE.2022-07-29T19-40-48Z
-    created: "2023-09-30T13:45:44.307480879-07:00"
+    created: "2024-01-12T10:18:32.8700854-08:00"
     description: Multi-Cloud Object Storage
     digest: d11db37963636922cb778b6bc0ad2ca4724cb391ea7b785995ada52467d7dd83
     home: https://min.io
@@ -531,7 +553,7 @@ entries:
     version: 4.0.8
   - apiVersion: v1
     appVersion: RELEASE.2022-07-26T00-53-03Z
-    created: "2023-09-30T13:45:44.306527164-07:00"
+    created: "2024-01-12T10:18:32.869222849-08:00"
     description: Multi-Cloud Object Storage
     digest: ca775e08c84331bb5029d4d29867d30c16e2c62e897788eb432212a756e91e4e
     home: https://min.io
@@ -553,7 +575,7 @@ entries:
     version: 4.0.7
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-09-30T13:45:44.305405238-07:00"
+    created: "2024-01-12T10:18:32.868312211-08:00"
     description: Multi-Cloud Object Storage
     digest: 06542b8f3d149d5908b15de9a8d6f8cf304af0213830be56dc315785d14f9ccd
     home: https://min.io
@@ -575,7 +597,7 @@ entries:
     version: 4.0.6
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-09-30T13:45:44.304317131-07:00"
+    created: "2024-01-12T10:18:32.867446886-08:00"
     description: Multi-Cloud Object Storage
     digest: dd2676362f067454a496cdd293609d0c904b08f521625af49f95402a024ba1f5
     home: https://min.io
@@ -597,7 +619,7 @@ entries:
     version: 4.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-09-30T13:45:44.303451898-07:00"
+    created: "2024-01-12T10:18:32.866527174-08:00"
     description: Multi-Cloud Object Storage
     digest: bab9ef192d4eda4c572ad0ce0cf551736c847f582d1837d6833ee10543c23167
     home: https://min.io
@@ -619,7 +641,7 @@ entries:
     version: 4.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-09-30T13:45:44.302546931-07:00"
+    created: "2024-01-12T10:18:32.864931103-08:00"
     description: Multi-Cloud Object Storage
     digest: c770bb9841c76576e4e8573f78b0ec33e0d729504c9667e67ad62d48df5ed64c
     home: https://min.io
@@ -641,7 +663,7 @@ entries:
     version: 4.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-09-30T13:45:44.301269526-07:00"
+    created: "2024-01-12T10:18:32.864042379-08:00"
     description: Multi-Cloud Object Storage
     digest: 95835f4199d963e2a23a2493610b348e6f2ff8b71c1a648c4a3b84af9b7a83eb
     home: https://min.io
@@ -663,7 +685,7 @@ entries:
     version: 4.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-04-30T22-23-53Z
-    created: "2023-09-30T13:45:44.29213066-07:00"
+    created: "2024-01-12T10:18:32.856372066-08:00"
     description: Multi-Cloud Object Storage
     digest: 55a088c403b056e1f055a97426aa11759c3d6cbad38face170fe6cbbec7d568f
     home: https://min.io
@@ -685,7 +707,7 @@ entries:
     version: 4.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-04-26T01-20-24Z
-    created: "2023-09-30T13:45:44.291284045-07:00"
+    created: "2024-01-12T10:18:32.855514009-08:00"
     description: Multi-Cloud Object Storage
     digest: f541237e24336ec3f7f45ae0d523fef694e3a2f9ef648c5b11c15734db6ba2b2
     home: https://min.io
@@ -707,7 +729,7 @@ entries:
     version: 4.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-04-16T04-26-02Z
-    created: "2023-09-30T13:45:44.290426627-07:00"
+    created: "2024-01-12T10:18:32.854654894-08:00"
     description: Multi-Cloud Object Storage
     digest: edc0c3dd6d5246a06b74ba16bb4aff80a6d7225dc9aecf064fd89a8af371b9c1
     home: https://min.io
@@ -729,7 +751,7 @@ entries:
     version: 3.6.6
   - apiVersion: v1
     appVersion: RELEASE.2022-04-12T06-55-35Z
-    created: "2023-09-30T13:45:44.289556731-07:00"
+    created: "2024-01-12T10:18:32.85377962-08:00"
     description: Multi-Cloud Object Storage
     digest: 211e89f6b9eb0b9a3583abaa127be60e1f9717a098e6b2858cb9dc1cc50c1650
     home: https://min.io
@@ -751,7 +773,7 @@ entries:
     version: 3.6.5
   - apiVersion: v1
     appVersion: RELEASE.2022-04-09T15-09-52Z
-    created: "2023-09-30T13:45:44.28851016-07:00"
+    created: "2024-01-12T10:18:32.852864945-08:00"
     description: Multi-Cloud Object Storage
     digest: 534a879d73b370a18b554b93d0930e1c115419619c4ce4ec7dbaae632acacf06
     home: https://min.io
@@ -773,7 +795,7 @@ entries:
     version: 3.6.4
   - apiVersion: v1
     appVersion: RELEASE.2022-03-24T00-43-44Z
-    created: "2023-09-30T13:45:44.287632827-07:00"
+    created: "2024-01-12T10:18:32.851420999-08:00"
     description: Multi-Cloud Object Storage
     digest: 99508b20eb0083a567dcccaf9a6c237e09575ed1d70cd2e8333f89c472d13d75
     home: https://min.io
@@ -795,7 +817,7 @@ entries:
     version: 3.6.3
   - apiVersion: v1
     appVersion: RELEASE.2022-03-17T06-34-49Z
-    created: "2023-09-30T13:45:44.28672558-07:00"
+    created: "2024-01-12T10:18:32.850379027-08:00"
     description: Multi-Cloud Object Storage
     digest: b4cd25611ca322b1d23d23112fdfa6b068fd91eefe0b0663b88ff87ea4282495
     home: https://min.io
@@ -817,7 +839,7 @@ entries:
     version: 3.6.2
   - apiVersion: v1
     appVersion: RELEASE.2022-03-14T18-25-24Z
-    created: "2023-09-30T13:45:44.285161842-07:00"
+    created: "2024-01-12T10:18:32.849503068-08:00"
     description: Multi-Cloud Object Storage
     digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c
     home: https://min.io
@@ -839,7 +861,7 @@ entries:
     version: 3.6.1
   - apiVersion: v1
     appVersion: RELEASE.2022-03-11T23-57-45Z
-    created: "2023-09-30T13:45:44.284217509-07:00"
+    created: "2024-01-12T10:18:32.848582714-08:00"
     description: Multi-Cloud Object Storage
     digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d
     home: https://min.io
@@ -861,7 +883,7 @@ entries:
     version: 3.6.0
   - apiVersion: v1
     appVersion: RELEASE.2022-03-03T21-21-16Z
-    created: "2023-09-30T13:45:44.28321379-07:00"
+    created: "2024-01-12T10:18:32.847695596-08:00"
     description: Multi-Cloud Object Storage
     digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
     home: https://min.io
@@ -883,7 +905,7 @@ entries:
     version: 3.5.9
   - apiVersion: v1
     appVersion: RELEASE.2022-02-26T02-54-46Z
-    created: "2023-09-30T13:45:44.282331055-07:00"
+    created: "2024-01-12T10:18:32.846787639-08:00"
     description: Multi-Cloud Object Storage
     digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
     home: https://min.io
@@ -905,7 +927,7 @@ entries:
     version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-09-30T13:45:44.281421196-07:00"
+    created: "2024-01-12T10:18:32.845713939-08:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -927,7 +949,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-09-30T13:45:44.280388028-07:00"
+    created: "2024-01-12T10:18:32.84414488-08:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -949,7 +971,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2023-09-30T13:45:44.279117054-07:00"
+    created: "2024-01-12T10:18:32.843276246-08:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -971,7 +993,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-09-30T13:45:44.277558085-07:00"
+    created: "2024-01-12T10:18:32.8424685-08:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -993,7 +1015,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-09-30T13:45:44.276727539-07:00"
+    created: "2024-01-12T10:18:32.841603809-08:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -1015,7 +1037,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2023-09-30T13:45:44.275957276-07:00"
+    created: "2024-01-12T10:18:32.840780358-08:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -1037,7 +1059,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-09-30T13:45:44.275214578-07:00"
+    created: "2024-01-12T10:18:32.839990751-08:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -1059,7 +1081,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-09-30T13:45:44.274471492-07:00"
+    created: "2024-01-12T10:18:32.83925152-08:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -1081,7 +1103,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2023-09-30T13:45:44.273733788-07:00"
+    created: "2024-01-12T10:18:32.838482583-08:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -1103,7 +1125,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2023-09-30T13:45:44.272981708-07:00"
+    created: "2024-01-12T10:18:32.83699659-08:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -1125,7 +1147,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2023-09-30T13:45:44.272032593-07:00"
+    created: "2024-01-12T10:18:32.836155167-08:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -1147,7 +1169,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2023-09-30T13:45:44.270594751-07:00"
+    created: "2024-01-12T10:18:32.835416975-08:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -1169,7 +1191,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2023-09-30T13:45:44.269865392-07:00"
+    created: "2024-01-12T10:18:32.834697219-08:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -1191,7 +1213,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-09-30T13:45:44.269127567-07:00"
+    created: "2024-01-12T10:18:32.833960249-08:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -1213,7 +1235,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-09-30T13:45:44.268163723-07:00"
+    created: "2024-01-12T10:18:32.83317557-08:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -1235,7 +1257,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-09-30T13:45:44.267246354-07:00"
+    created: "2024-01-12T10:18:32.832262381-08:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -1257,7 +1279,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2023-09-30T13:45:44.266343771-07:00"
+    created: "2024-01-12T10:18:32.830707536-08:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -1279,7 +1301,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-09-30T13:45:44.265456326-07:00"
+    created: "2024-01-12T10:18:32.829956619-08:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -1301,7 +1323,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-09-30T13:45:44.264304002-07:00"
+    created: "2024-01-12T10:18:32.829241149-08:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -1323,7 +1345,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-09-30T13:45:44.259435328-07:00"
+    created: "2024-01-12T10:18:32.8284866-08:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -1345,7 +1367,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-09-30T13:45:44.258448201-07:00"
+    created: "2024-01-12T10:18:32.827780599-08:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -1367,7 +1389,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-09-30T13:45:44.257734262-07:00"
+    created: "2024-01-12T10:18:32.827068512-08:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -1389,7 +1411,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2023-09-30T13:45:44.257013636-07:00"
+    created: "2024-01-12T10:18:32.826266888-08:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -1411,7 +1433,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2023-09-30T13:45:44.25605328-07:00"
+    created: "2024-01-12T10:18:32.821671133-08:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -1433,7 +1455,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2023-09-30T13:45:44.255324762-07:00"
+    created: "2024-01-12T10:18:32.820956761-08:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -1455,7 +1477,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2023-09-30T13:45:44.254617009-07:00"
+    created: "2024-01-12T10:18:32.82017685-08:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -1477,7 +1499,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-09-30T13:45:44.253895027-07:00"
+    created: "2024-01-12T10:18:32.819488951-08:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -1499,7 +1521,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-09-30T13:45:44.252741769-07:00"
+    created: "2024-01-12T10:18:32.818789579-08:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -1521,7 +1543,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-09-30T13:45:44.251426132-07:00"
+    created: "2024-01-12T10:18:32.818088961-08:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -1543,7 +1565,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-09-30T13:45:44.2504309-07:00"
+    created: "2024-01-12T10:18:32.817375073-08:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -1565,7 +1587,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-09-30T13:45:44.249671479-07:00"
+    created: "2024-01-12T10:18:32.815865481-08:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -1587,7 +1609,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-09-30T13:45:44.248904262-07:00"
+    created: "2024-01-12T10:18:32.815102352-08:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -1609,7 +1631,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-09-30T13:45:44.248011769-07:00"
+    created: "2024-01-12T10:18:32.814342534-08:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -1631,7 +1653,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2023-09-30T13:45:44.246954326-07:00"
+    created: "2024-01-12T10:18:32.813578506-08:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -1653,7 +1675,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-09-30T13:45:44.246078126-07:00"
+    created: "2024-01-12T10:18:32.812809365-08:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -1675,7 +1697,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-09-30T13:45:44.24452265-07:00"
+    created: "2024-01-12T10:18:32.812019194-08:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -1697,7 +1719,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-09-30T13:45:44.243841906-07:00"
+    created: "2024-01-12T10:18:32.811220435-08:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -1717,7 +1739,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-09-30T13:45:44.242951969-07:00"
+    created: "2024-01-12T10:18:32.810561305-08:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -1737,7 +1759,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-09-30T13:45:44.242280535-07:00"
+    created: "2024-01-12T10:18:32.809892316-08:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -1757,7 +1779,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-09-30T13:45:44.241614992-07:00"
+    created: "2024-01-12T10:18:32.809237276-08:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -1777,7 +1799,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-09-30T13:45:44.24052836-07:00"
+    created: "2024-01-12T10:18:32.808472922-08:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -1797,7 +1819,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-09-30T13:45:44.23955753-07:00"
+    created: "2024-01-12T10:18:32.807127702-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -1817,7 +1839,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-09-30T13:45:44.234084367-07:00"
+    created: "2024-01-12T10:18:32.806306069-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -1837,7 +1859,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2023-09-30T13:45:44.233235408-07:00"
+    created: "2024-01-12T10:18:32.805521835-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -1855,4 +1877,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2023-09-30T13:45:44.232215184-07:00"
+generated: "2024-01-12T10:18:32.804598259-08:00"

internal/http/headers.go

@@ -227,6 +227,10 @@ const (
 	MinIOSourceObjectRetentionTimestamp = "X-Minio-Source-Replication-Retention-Timestamp"
 	// Header indiicates last rtention update time on source
 	MinIOSourceObjectLegalHoldTimestamp = "X-Minio-Source-Replication-LegalHold-Timestamp"
+	// Header indicates a Tag operation was performed on one/more peers successfully, though the
+	// current cluster does not have the object yet. This is in a site/bucket replication scenario.
+	MinIOTaggingProxied = "X-Minio-Tagging-Proxied"
+
 	// predicted date/time of transition
 	MinIOTransition            = "X-Minio-Transition"
 	MinIOLifecycleCfgUpdatedAt = "X-Minio-LifecycleConfig-UpdatedAt"