mirror of
https://github.com/pgsty/minio.git
synced 2026-10-31 04:50:40 +03:00
feat(ilm): relocate hot objects across server pools by GET frequency
Keep NVMe/HDD pool pairs useful without remote tiering: promote objects that are read often, demote previously moved objects once they go idle, and leave the feature off until operators set a two-pool topology. Signed-off-by: mr javad seydi <seydi.birjand@gmail.com>
This commit is contained in:
@@ -39,6 +39,7 @@ const (
|
|||||||
lcEventSrc_s3PutObject
|
lcEventSrc_s3PutObject
|
||||||
lcEventSrc_s3CopyObject
|
lcEventSrc_s3CopyObject
|
||||||
lcEventSrc_s3CompleteMultipartUpload
|
lcEventSrc_s3CompleteMultipartUpload
|
||||||
|
lcEventSrc_AccessTier
|
||||||
)
|
)
|
||||||
|
|
||||||
//revive:enable:var-naming
|
//revive:enable:var-naming
|
||||||
|
|||||||
@@ -227,7 +227,7 @@ func initHelp() {
|
|||||||
},
|
},
|
||||||
config.HelpKV{
|
config.HelpKV{
|
||||||
Key: config.ILMSubSys,
|
Key: config.ILMSubSys,
|
||||||
Description: "manage ILM settings for expiration and transition workers",
|
Description: "manage ILM settings for expiration, transition, and access-tier workers",
|
||||||
Optional: true,
|
Optional: true,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@@ -704,6 +704,9 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
|
|||||||
if globalExpiryState != nil {
|
if globalExpiryState != nil {
|
||||||
globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
|
globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers)
|
||||||
}
|
}
|
||||||
|
if globalAccessTierState != nil {
|
||||||
|
globalAccessTierState.UpdateWorkers(ilmCfg.AccessWorkers)
|
||||||
|
}
|
||||||
globalILMConfig.update(ilmCfg)
|
globalILMConfig.update(ilmCfg)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -899,6 +899,7 @@ type scannerItem struct {
|
|||||||
objectName string // Only the object name without prefixes.
|
objectName string // Only the object name without prefixes.
|
||||||
replication replicationConfig
|
replication replicationConfig
|
||||||
lifeCycle *lifecycle.Lifecycle
|
lifeCycle *lifecycle.Lifecycle
|
||||||
|
poolIdx int
|
||||||
Typ fs.FileMode
|
Typ fs.FileMode
|
||||||
heal struct {
|
heal struct {
|
||||||
enabled bool
|
enabled bool
|
||||||
@@ -909,6 +910,7 @@ type scannerItem struct {
|
|||||||
|
|
||||||
type sizeSummary struct {
|
type sizeSummary struct {
|
||||||
totalSize int64
|
totalSize int64
|
||||||
|
hotTierSize int64
|
||||||
versions uint64
|
versions uint64
|
||||||
deleteMarkers uint64
|
deleteMarkers uint64
|
||||||
replicatedSize int64
|
replicatedSize int64
|
||||||
@@ -1159,6 +1161,14 @@ eventLoop:
|
|||||||
globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents)
|
globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents)
|
||||||
}
|
}
|
||||||
i.alertExcessiveVersions(remainingVersions, cumulativeSize)
|
i.alertExcessiveVersions(remainingVersions, cumulativeSize)
|
||||||
|
if globalILMConfig.accessTieringEnabled() {
|
||||||
|
for idx, oi := range objInfos {
|
||||||
|
if oi.IsLatest && events[idx].Action == lifecycle.NoneAction {
|
||||||
|
applyAccessTransition(ctx, i, oi)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
|
func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
|
||||||
@@ -1474,6 +1484,8 @@ const (
|
|||||||
ILMFreeVersionDelete = "ilm:free-version-delete"
|
ILMFreeVersionDelete = "ilm:free-version-delete"
|
||||||
// ILMTransition - audit trail for ILM transitioning.
|
// ILMTransition - audit trail for ILM transitioning.
|
||||||
ILMTransition = " ilm:transition"
|
ILMTransition = " ilm:transition"
|
||||||
|
// ILMAccessTier - audit trail for moving objects between server pools.
|
||||||
|
ILMAccessTier = "ilm:access-tier"
|
||||||
)
|
)
|
||||||
|
|
||||||
func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
|
func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) {
|
||||||
@@ -1485,6 +1497,8 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
|
|||||||
apiName = "ILMFreeVersionDelete"
|
apiName = "ILMFreeVersionDelete"
|
||||||
case ILMTransition:
|
case ILMTransition:
|
||||||
apiName = "ILMTransition"
|
apiName = "ILMTransition"
|
||||||
|
case ILMAccessTier:
|
||||||
|
apiName = "ILMAccessTier"
|
||||||
}
|
}
|
||||||
auditLogInternal(ctx, AuditLogOptions{
|
auditLogInternal(ctx, AuditLogOptions{
|
||||||
Event: event,
|
Event: event,
|
||||||
|
|||||||
+60
-5
@@ -61,6 +61,7 @@ type dataUsageEntry struct {
|
|||||||
Children dataUsageHashMap `msg:"ch"`
|
Children dataUsageHashMap `msg:"ch"`
|
||||||
// These fields do no include any children.
|
// These fields do no include any children.
|
||||||
Size int64 `msg:"sz"`
|
Size int64 `msg:"sz"`
|
||||||
|
HotTierSize int64 `msg:"hts"`
|
||||||
Objects uint64 `msg:"os"`
|
Objects uint64 `msg:"os"`
|
||||||
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
Versions uint64 `msg:"vs"` // Versions that are not delete markers.
|
||||||
DeleteMarkers uint64 `msg:"dms"`
|
DeleteMarkers uint64 `msg:"dms"`
|
||||||
@@ -134,8 +135,8 @@ func (ts tierStats) add(u tierStats) tierStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
|
//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
|
||||||
//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7
|
//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 dataUsageEntryV8
|
||||||
|
|
||||||
//msgp:tuple dataUsageEntryV2
|
//msgp:tuple dataUsageEntryV2
|
||||||
type dataUsageEntryV2 struct {
|
type dataUsageEntryV2 struct {
|
||||||
@@ -199,14 +200,30 @@ type dataUsageEntryV7 struct {
|
|||||||
Compacted bool `msg:"c"`
|
Compacted bool `msg:"c"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dataUsageEntryV8 is the on-disk shape before access-tier accounting was
|
||||||
|
// introduced. Keep it so caches written by the previous release decode
|
||||||
|
// without being discarded.
|
||||||
|
type dataUsageEntryV8 struct {
|
||||||
|
Children dataUsageHashMap `msg:"ch"`
|
||||||
|
// These fields do no include any children.
|
||||||
|
Size int64 `msg:"sz"`
|
||||||
|
Objects uint64 `msg:"os"`
|
||||||
|
Versions uint64 `msg:"vs"`
|
||||||
|
DeleteMarkers uint64 `msg:"dms"`
|
||||||
|
ObjSizes sizeHistogram `msg:"szs"`
|
||||||
|
ObjVersions versionsHistogram `msg:"vh"`
|
||||||
|
AllTierStats *allTierStats `msg:"ats,omitempty"`
|
||||||
|
Compacted bool `msg:"c"`
|
||||||
|
}
|
||||||
|
|
||||||
// dataUsageCache contains a cache of data usage entries latest version.
|
// dataUsageCache contains a cache of data usage entries latest version.
|
||||||
type dataUsageCache struct {
|
type dataUsageCache struct {
|
||||||
Info dataUsageCacheInfo
|
Info dataUsageCacheInfo
|
||||||
Cache map[string]dataUsageEntry
|
Cache map[string]dataUsageEntry
|
||||||
}
|
}
|
||||||
|
|
||||||
//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
|
//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
|
||||||
//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7
|
//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 dataUsageCacheV8
|
||||||
|
|
||||||
// dataUsageCacheV2 contains a cache of data usage entries version 2.
|
// dataUsageCacheV2 contains a cache of data usage entries version 2.
|
||||||
type dataUsageCacheV2 struct {
|
type dataUsageCacheV2 struct {
|
||||||
@@ -244,6 +261,12 @@ type dataUsageCacheV7 struct {
|
|||||||
Cache map[string]dataUsageEntryV7
|
Cache map[string]dataUsageEntryV7
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// dataUsageCacheV8 contains a cache of data usage entries version 8.
|
||||||
|
type dataUsageCacheV8 struct {
|
||||||
|
Info dataUsageCacheInfo
|
||||||
|
Cache map[string]dataUsageEntryV8
|
||||||
|
}
|
||||||
|
|
||||||
//msgp:ignore dataUsageEntryInfo
|
//msgp:ignore dataUsageEntryInfo
|
||||||
type dataUsageEntryInfo struct {
|
type dataUsageEntryInfo struct {
|
||||||
Name string
|
Name string
|
||||||
@@ -272,6 +295,7 @@ type dataUsageCacheInfo struct {
|
|||||||
|
|
||||||
func (e *dataUsageEntry) addSizes(summary sizeSummary) {
|
func (e *dataUsageEntry) addSizes(summary sizeSummary) {
|
||||||
e.Size += summary.totalSize
|
e.Size += summary.totalSize
|
||||||
|
e.HotTierSize += summary.hotTierSize
|
||||||
e.Versions += summary.versions
|
e.Versions += summary.versions
|
||||||
e.DeleteMarkers += summary.deleteMarkers
|
e.DeleteMarkers += summary.deleteMarkers
|
||||||
e.ObjSizes.add(summary.totalSize)
|
e.ObjSizes.add(summary.totalSize)
|
||||||
@@ -291,6 +315,7 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
|
|||||||
e.Versions += other.Versions
|
e.Versions += other.Versions
|
||||||
e.DeleteMarkers += other.DeleteMarkers
|
e.DeleteMarkers += other.DeleteMarkers
|
||||||
e.Size += other.Size
|
e.Size += other.Size
|
||||||
|
e.HotTierSize += other.HotTierSize
|
||||||
|
|
||||||
for i, v := range other.ObjSizes[:] {
|
for i, v := range other.ObjSizes[:] {
|
||||||
e.ObjSizes[i] += v
|
e.ObjSizes[i] += v
|
||||||
@@ -431,6 +456,7 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
|
|||||||
flat := d.flatten(*e)
|
flat := d.flatten(*e)
|
||||||
dui := DataUsageInfo{
|
dui := DataUsageInfo{
|
||||||
LastUpdate: d.Info.LastUpdate,
|
LastUpdate: d.Info.LastUpdate,
|
||||||
|
ScannerCycle: d.Info.NextCycle,
|
||||||
ObjectsTotalCount: flat.Objects,
|
ObjectsTotalCount: flat.Objects,
|
||||||
VersionsTotalCount: flat.Versions,
|
VersionsTotalCount: flat.Versions,
|
||||||
DeleteMarkersTotalCount: flat.DeleteMarkers,
|
DeleteMarkersTotalCount: flat.DeleteMarkers,
|
||||||
@@ -781,6 +807,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
|
|||||||
flat := d.flatten(*e)
|
flat := d.flatten(*e)
|
||||||
bui := BucketUsageInfo{
|
bui := BucketUsageInfo{
|
||||||
Size: uint64(flat.Size),
|
Size: uint64(flat.Size),
|
||||||
|
HotTierSize: uint64(max(flat.HotTierSize, 0)),
|
||||||
VersionsCount: flat.Versions,
|
VersionsCount: flat.Versions,
|
||||||
ObjectsCount: flat.Objects,
|
ObjectsCount: flat.Objects,
|
||||||
DeleteMarkersCount: flat.DeleteMarkers,
|
DeleteMarkersCount: flat.DeleteMarkers,
|
||||||
@@ -980,7 +1007,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
|||||||
// Bumping the cache version will drop data from previous versions
|
// Bumping the cache version will drop data from previous versions
|
||||||
// and write new data with the new version.
|
// and write new data with the new version.
|
||||||
const (
|
const (
|
||||||
dataUsageCacheVerCurrent = 8
|
dataUsageCacheVerCurrent = 9
|
||||||
|
dataUsageCacheVerV8 = 8
|
||||||
dataUsageCacheVerV7 = 7
|
dataUsageCacheVerV7 = 7
|
||||||
dataUsageCacheVerV6 = 6
|
dataUsageCacheVerV6 = 6
|
||||||
dataUsageCacheVerV5 = 5
|
dataUsageCacheVerV5 = 5
|
||||||
@@ -1181,6 +1209,33 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
case dataUsageCacheVerV8:
|
||||||
|
// Zstd compressed.
|
||||||
|
dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer dec.Close()
|
||||||
|
dold := &dataUsageCacheV8{}
|
||||||
|
if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
d.Info = dold.Info
|
||||||
|
d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
|
||||||
|
for k, v := range dold.Cache {
|
||||||
|
d.Cache[k] = dataUsageEntry{
|
||||||
|
Children: v.Children,
|
||||||
|
Size: v.Size,
|
||||||
|
Objects: v.Objects,
|
||||||
|
Versions: v.Versions,
|
||||||
|
DeleteMarkers: v.DeleteMarkers,
|
||||||
|
ObjSizes: v.ObjSizes,
|
||||||
|
ObjVersions: v.ObjVersions,
|
||||||
|
AllTierStats: v.AllTierStats,
|
||||||
|
Compacted: v.Compacted,
|
||||||
|
}
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
case dataUsageCacheVerCurrent:
|
case dataUsageCacheVerCurrent:
|
||||||
// Zstd compressed.
|
// Zstd compressed.
|
||||||
|
|||||||
+605
-9
@@ -1591,6 +1591,145 @@ func (z *dataUsageCacheV7) Msgsize() (s int) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *dataUsageCacheV8) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "Info":
|
||||||
|
err = z.Info.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Info")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Cache":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Cache")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if z.Cache == nil {
|
||||||
|
z.Cache = make(map[string]dataUsageEntryV8, zb0002)
|
||||||
|
} else if len(z.Cache) > 0 {
|
||||||
|
clear(z.Cache)
|
||||||
|
}
|
||||||
|
for zb0002 > 0 {
|
||||||
|
zb0002--
|
||||||
|
var za0001 string
|
||||||
|
za0001, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Cache")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var za0002 dataUsageEntryV8
|
||||||
|
err = za0002.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Cache", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.Cache[za0001] = za0002
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *dataUsageCacheV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "Info":
|
||||||
|
bts, err = z.Info.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Info")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "Cache":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Cache")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if z.Cache == nil {
|
||||||
|
z.Cache = make(map[string]dataUsageEntryV8, zb0002)
|
||||||
|
} else if len(z.Cache) > 0 {
|
||||||
|
clear(z.Cache)
|
||||||
|
}
|
||||||
|
for zb0002 > 0 {
|
||||||
|
var za0002 dataUsageEntryV8
|
||||||
|
zb0002--
|
||||||
|
var za0001 string
|
||||||
|
za0001, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Cache")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
bts, err = za0002.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Cache", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.Cache[za0001] = za0002
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z *dataUsageCacheV8) Msgsize() (s int) {
|
||||||
|
s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize
|
||||||
|
if z.Cache != nil {
|
||||||
|
for za0001, za0002 := range z.Cache {
|
||||||
|
_ = za0002
|
||||||
|
s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// DecodeMsg implements msgp.Decodable
|
// DecodeMsg implements msgp.Decodable
|
||||||
func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
var field []byte
|
var field []byte
|
||||||
@@ -1623,6 +1762,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
err = msgp.WrapError(err, "Size")
|
err = msgp.WrapError(err, "Size")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
case "hts":
|
||||||
|
z.HotTierSize, err = dc.ReadInt64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "HotTierSize")
|
||||||
|
return
|
||||||
|
}
|
||||||
case "os":
|
case "os":
|
||||||
z.Objects, err = dc.ReadUint64()
|
z.Objects, err = dc.ReadUint64()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1801,12 +1946,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
|||||||
// EncodeMsg implements msgp.Encodable
|
// EncodeMsg implements msgp.Encodable
|
||||||
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
// check for omitted fields
|
// check for omitted fields
|
||||||
zb0001Len := uint32(9)
|
zb0001Len := uint32(10)
|
||||||
var zb0001Mask uint16 /* 9 bits */
|
var zb0001Mask uint16 /* 10 bits */
|
||||||
_ = zb0001Mask
|
_ = zb0001Mask
|
||||||
if z.AllTierStats == nil {
|
if z.AllTierStats == nil {
|
||||||
zb0001Len--
|
zb0001Len--
|
||||||
zb0001Mask |= 0x80
|
zb0001Mask |= 0x100
|
||||||
}
|
}
|
||||||
// variable map header, size zb0001Len
|
// variable map header, size zb0001Len
|
||||||
err = en.Append(0x80 | uint8(zb0001Len))
|
err = en.Append(0x80 | uint8(zb0001Len))
|
||||||
@@ -1836,6 +1981,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
|||||||
err = msgp.WrapError(err, "Size")
|
err = msgp.WrapError(err, "Size")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
// write "hts"
|
||||||
|
err = en.Append(0xa3, 0x68, 0x74, 0x73)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt64(z.HotTierSize)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "HotTierSize")
|
||||||
|
return
|
||||||
|
}
|
||||||
// write "os"
|
// write "os"
|
||||||
err = en.Append(0xa2, 0x6f, 0x73)
|
err = en.Append(0xa2, 0x6f, 0x73)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1900,7 +2055,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (zb0001Mask & 0x80) == 0 { // if not omitted
|
if (zb0001Mask & 0x100) == 0 { // if not omitted
|
||||||
// write "ats"
|
// write "ats"
|
||||||
err = en.Append(0xa3, 0x61, 0x74, 0x73)
|
err = en.Append(0xa3, 0x61, 0x74, 0x73)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1981,12 +2136,12 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
|||||||
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
o = msgp.Require(b, z.Msgsize())
|
o = msgp.Require(b, z.Msgsize())
|
||||||
// check for omitted fields
|
// check for omitted fields
|
||||||
zb0001Len := uint32(9)
|
zb0001Len := uint32(10)
|
||||||
var zb0001Mask uint16 /* 9 bits */
|
var zb0001Mask uint16 /* 10 bits */
|
||||||
_ = zb0001Mask
|
_ = zb0001Mask
|
||||||
if z.AllTierStats == nil {
|
if z.AllTierStats == nil {
|
||||||
zb0001Len--
|
zb0001Len--
|
||||||
zb0001Mask |= 0x80
|
zb0001Mask |= 0x100
|
||||||
}
|
}
|
||||||
// variable map header, size zb0001Len
|
// variable map header, size zb0001Len
|
||||||
o = append(o, 0x80|uint8(zb0001Len))
|
o = append(o, 0x80|uint8(zb0001Len))
|
||||||
@@ -2003,6 +2158,9 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
|||||||
// string "sz"
|
// string "sz"
|
||||||
o = append(o, 0xa2, 0x73, 0x7a)
|
o = append(o, 0xa2, 0x73, 0x7a)
|
||||||
o = msgp.AppendInt64(o, z.Size)
|
o = msgp.AppendInt64(o, z.Size)
|
||||||
|
// string "hts"
|
||||||
|
o = append(o, 0xa3, 0x68, 0x74, 0x73)
|
||||||
|
o = msgp.AppendInt64(o, z.HotTierSize)
|
||||||
// string "os"
|
// string "os"
|
||||||
o = append(o, 0xa2, 0x6f, 0x73)
|
o = append(o, 0xa2, 0x6f, 0x73)
|
||||||
o = msgp.AppendUint64(o, z.Objects)
|
o = msgp.AppendUint64(o, z.Objects)
|
||||||
@@ -2024,7 +2182,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
|||||||
for za0002 := range z.ObjVersions {
|
for za0002 := range z.ObjVersions {
|
||||||
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
|
o = msgp.AppendUint64(o, z.ObjVersions[za0002])
|
||||||
}
|
}
|
||||||
if (zb0001Mask & 0x80) == 0 { // if not omitted
|
if (zb0001Mask & 0x100) == 0 { // if not omitted
|
||||||
// string "ats"
|
// string "ats"
|
||||||
o = append(o, 0xa3, 0x61, 0x74, 0x73)
|
o = append(o, 0xa3, 0x61, 0x74, 0x73)
|
||||||
if z.AllTierStats == nil {
|
if z.AllTierStats == nil {
|
||||||
@@ -2088,6 +2246,12 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
err = msgp.WrapError(err, "Size")
|
err = msgp.WrapError(err, "Size")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
case "hts":
|
||||||
|
z.HotTierSize, bts, err = msgp.ReadInt64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "HotTierSize")
|
||||||
|
return
|
||||||
|
}
|
||||||
case "os":
|
case "os":
|
||||||
z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
|
z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -2265,7 +2429,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
|||||||
|
|
||||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
func (z *dataUsageEntry) Msgsize() (s int) {
|
func (z *dataUsageEntry) Msgsize() (s int) {
|
||||||
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
|
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
|
||||||
if z.AllTierStats == nil {
|
if z.AllTierStats == nil {
|
||||||
s += msgp.NilSize
|
s += msgp.NilSize
|
||||||
} else {
|
} else {
|
||||||
@@ -3258,6 +3422,438 @@ func (z *dataUsageEntryV7) Msgsize() (s int) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *dataUsageEntryV8) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var zb0001Mask uint8 /* 1 bits */
|
||||||
|
_ = zb0001Mask
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "ch":
|
||||||
|
err = z.Children.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Children")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "sz":
|
||||||
|
z.Size, err = dc.ReadInt64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Size")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "os":
|
||||||
|
z.Objects, err = dc.ReadUint64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Objects")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "vs":
|
||||||
|
z.Versions, err = dc.ReadUint64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Versions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "dms":
|
||||||
|
z.DeleteMarkers, err = dc.ReadUint64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "DeleteMarkers")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "szs":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, err = dc.ReadArrayHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjSizes")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if zb0002 != uint32(dataUsageBucketLen) {
|
||||||
|
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0001 := range z.ObjSizes {
|
||||||
|
z.ObjSizes[za0001], err = dc.ReadUint64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjSizes", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "vh":
|
||||||
|
var zb0003 uint32
|
||||||
|
zb0003, err = dc.ReadArrayHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if zb0003 != uint32(dataUsageVersionLen) {
|
||||||
|
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0002 := range z.ObjVersions {
|
||||||
|
z.ObjVersions[za0002], err = dc.ReadUint64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "ats":
|
||||||
|
if dc.IsNil() {
|
||||||
|
err = dc.ReadNil()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.AllTierStats = nil
|
||||||
|
} else {
|
||||||
|
if z.AllTierStats == nil {
|
||||||
|
z.AllTierStats = new(allTierStats)
|
||||||
|
}
|
||||||
|
var zb0004 uint32
|
||||||
|
zb0004, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0004 > 0 {
|
||||||
|
zb0004--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "ts":
|
||||||
|
var zb0005 uint32
|
||||||
|
zb0005, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if z.AllTierStats.Tiers == nil {
|
||||||
|
z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
|
||||||
|
} else if len(z.AllTierStats.Tiers) > 0 {
|
||||||
|
clear(z.AllTierStats.Tiers)
|
||||||
|
}
|
||||||
|
for zb0005 > 0 {
|
||||||
|
zb0005--
|
||||||
|
var za0003 string
|
||||||
|
za0003, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var za0004 tierStats
|
||||||
|
var zb0006 uint32
|
||||||
|
zb0006, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0006 > 0 {
|
||||||
|
zb0006--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "ts":
|
||||||
|
za0004.TotalSize, err = dc.ReadUint64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "nv":
|
||||||
|
za0004.NumVersions, err = dc.ReadInt()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "no":
|
||||||
|
za0004.NumObjects, err = dc.ReadInt()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
z.AllTierStats.Tiers[za0003] = za0004
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
zb0001Mask |= 0x1
|
||||||
|
case "c":
|
||||||
|
z.Compacted, err = dc.ReadBool()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Compacted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Clear omitted fields.
|
||||||
|
if (zb0001Mask & 0x1) == 0 {
|
||||||
|
z.AllTierStats = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *dataUsageEntryV8) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var zb0001Mask uint8 /* 1 bits */
|
||||||
|
_ = zb0001Mask
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "ch":
|
||||||
|
bts, err = z.Children.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Children")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "sz":
|
||||||
|
z.Size, bts, err = msgp.ReadInt64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Size")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "os":
|
||||||
|
z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Objects")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "vs":
|
||||||
|
z.Versions, bts, err = msgp.ReadUint64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Versions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "dms":
|
||||||
|
z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "DeleteMarkers")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "szs":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjSizes")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if zb0002 != uint32(dataUsageBucketLen) {
|
||||||
|
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0001 := range z.ObjSizes {
|
||||||
|
z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjSizes", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "vh":
|
||||||
|
var zb0003 uint32
|
||||||
|
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if zb0003 != uint32(dataUsageVersionLen) {
|
||||||
|
err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0002 := range z.ObjVersions {
|
||||||
|
z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "ObjVersions", za0002)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "ats":
|
||||||
|
if msgp.IsNil(bts) {
|
||||||
|
bts, err = msgp.ReadNilBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.AllTierStats = nil
|
||||||
|
} else {
|
||||||
|
if z.AllTierStats == nil {
|
||||||
|
z.AllTierStats = new(allTierStats)
|
||||||
|
}
|
||||||
|
var zb0004 uint32
|
||||||
|
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0004 > 0 {
|
||||||
|
zb0004--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "ts":
|
||||||
|
var zb0005 uint32
|
||||||
|
zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if z.AllTierStats.Tiers == nil {
|
||||||
|
z.AllTierStats.Tiers = make(map[string]tierStats, zb0005)
|
||||||
|
} else if len(z.AllTierStats.Tiers) > 0 {
|
||||||
|
clear(z.AllTierStats.Tiers)
|
||||||
|
}
|
||||||
|
for zb0005 > 0 {
|
||||||
|
var za0004 tierStats
|
||||||
|
zb0005--
|
||||||
|
var za0003 string
|
||||||
|
za0003, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var zb0006 uint32
|
||||||
|
zb0006, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0006 > 0 {
|
||||||
|
zb0006--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "ts":
|
||||||
|
za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "nv":
|
||||||
|
za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "no":
|
||||||
|
za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
z.AllTierStats.Tiers[za0003] = za0004
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "AllTierStats")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
zb0001Mask |= 0x1
|
||||||
|
case "c":
|
||||||
|
z.Compacted, bts, err = msgp.ReadBoolBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Compacted")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Clear omitted fields.
|
||||||
|
if (zb0001Mask & 0x1) == 0 {
|
||||||
|
z.AllTierStats = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z *dataUsageEntryV8) Msgsize() (s int) {
|
||||||
|
s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4
|
||||||
|
if z.AllTierStats == nil {
|
||||||
|
s += msgp.NilSize
|
||||||
|
} else {
|
||||||
|
s += 1 + 3 + msgp.MapHeaderSize
|
||||||
|
if z.AllTierStats.Tiers != nil {
|
||||||
|
for za0003, za0004 := range z.AllTierStats.Tiers {
|
||||||
|
_ = za0004
|
||||||
|
s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s += 2 + msgp.BoolSize
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
// DecodeMsg implements msgp.Decodable
|
// DecodeMsg implements msgp.Decodable
|
||||||
func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
|
func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
{
|
{
|
||||||
|
|||||||
@@ -46,7 +46,8 @@ type BucketTargetUsageInfo struct {
|
|||||||
// - total objects in a bucket
|
// - total objects in a bucket
|
||||||
// - object size histogram per bucket
|
// - object size histogram per bucket
|
||||||
type BucketUsageInfo struct {
|
type BucketUsageInfo struct {
|
||||||
Size uint64 `json:"size"`
|
Size uint64 `json:"size"`
|
||||||
|
HotTierSize uint64 `json:"hotTierSize,omitempty"`
|
||||||
// Following five fields suffixed with V1 are here for backward compatibility
|
// Following five fields suffixed with V1 are here for backward compatibility
|
||||||
// Total Size for objects that have not yet been replicated
|
// Total Size for objects that have not yet been replicated
|
||||||
ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
|
ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"`
|
||||||
@@ -78,6 +79,10 @@ type DataUsageInfo struct {
|
|||||||
// LastUpdate is the timestamp of when the data usage info was last updated.
|
// LastUpdate is the timestamp of when the data usage info was last updated.
|
||||||
// This does not indicate a full scan.
|
// This does not indicate a full scan.
|
||||||
LastUpdate time.Time `json:"lastUpdate"`
|
LastUpdate time.Time `json:"lastUpdate"`
|
||||||
|
// ScannerCycle changes only after a complete scanner pass. Background
|
||||||
|
// consumers use it to distinguish a partial cache update from a baseline
|
||||||
|
// that has visited every bucket and server pool.
|
||||||
|
ScannerCycle uint32 `json:"scannerCycle,omitempty"`
|
||||||
|
|
||||||
// Objects total count across all buckets
|
// Objects total count across all buckets
|
||||||
ObjectsTotalCount uint64 `json:"objectsCount"`
|
ObjectsTotalCount uint64 `json:"objectsCount"`
|
||||||
|
|||||||
+95
-22
@@ -615,7 +615,17 @@ func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucke
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
|
func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) {
|
||||||
|
if dstPoolIdx != nil {
|
||||||
|
if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
|
||||||
|
return -1, errInvalidArgument
|
||||||
|
}
|
||||||
|
if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
|
||||||
|
return -1, toObjectErr(errDiskFull)
|
||||||
|
}
|
||||||
|
return *dstPoolIdx, nil
|
||||||
|
}
|
||||||
|
|
||||||
idx, err = z.getPoolIdxExistingNoLock(ctx, bucket, object)
|
idx, err = z.getPoolIdxExistingNoLock(ctx, bucket, object)
|
||||||
if err != nil && !isErrObjectNotFound(err) {
|
if err != nil && !isErrObjectNotFound(err) {
|
||||||
return idx, err
|
return idx, err
|
||||||
@@ -634,7 +644,17 @@ func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, objec
|
|||||||
// getPoolIdx returns the found previous object and its corresponding pool idx,
|
// getPoolIdx returns the found previous object and its corresponding pool idx,
|
||||||
// if none are found falls back to most available space pool, this function is
|
// if none are found falls back to most available space pool, this function is
|
||||||
// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
|
// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload.
|
||||||
func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
|
func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64, dstPoolIdx *int) (idx int, err error) {
|
||||||
|
if dstPoolIdx != nil {
|
||||||
|
if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
|
||||||
|
return -1, errInvalidArgument
|
||||||
|
}
|
||||||
|
if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
|
||||||
|
return -1, toObjectErr(errDiskFull)
|
||||||
|
}
|
||||||
|
return *dstPoolIdx, nil
|
||||||
|
}
|
||||||
|
|
||||||
pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{
|
pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{
|
||||||
SkipDecommissioned: true,
|
SkipDecommissioned: true,
|
||||||
SkipRebalancing: true,
|
SkipRebalancing: true,
|
||||||
@@ -656,6 +676,13 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri
|
|||||||
return idx, nil
|
return idx, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func dataMovementDstPool(opts ObjectOptions) *int {
|
||||||
|
if !opts.DataMovement {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return opts.DstPoolIdx
|
||||||
|
}
|
||||||
|
|
||||||
func (z *erasureServerPools) Shutdown(ctx context.Context) error {
|
func (z *erasureServerPools) Shutdown(ctx context.Context) error {
|
||||||
g := errgroup.WithNErrs(len(z.serverPools))
|
g := errgroup.WithNErrs(len(z.serverPools))
|
||||||
|
|
||||||
@@ -1090,14 +1117,20 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
|||||||
|
|
||||||
object = encodeDirObject(object)
|
object = encodeDirObject(object)
|
||||||
if z.SinglePool() {
|
if z.SinglePool() {
|
||||||
_, err := z.getPoolIdx(ctx, bucket, object, data.Size())
|
idx, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ObjectInfo{}, err
|
return ObjectInfo{}, err
|
||||||
}
|
}
|
||||||
|
if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx {
|
||||||
|
return ObjectInfo{}, DataMovementOverwriteErr{
|
||||||
|
Bucket: bucket, Object: object, VersionID: opts.VersionID,
|
||||||
|
Err: errDataMovementSrcDstPoolSame,
|
||||||
|
}
|
||||||
|
}
|
||||||
return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
|
return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
idx, err := z.getPoolIdx(ctx, bucket, object, data.Size())
|
idx, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ObjectInfo{}, err
|
return ObjectInfo{}, err
|
||||||
}
|
}
|
||||||
@@ -1145,6 +1178,28 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
|||||||
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
|
return ObjectInfo{}, z.deletePrefix(ctx, bucket, object)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Access-tier moves must recreate delete markers on the explicitly
|
||||||
|
// selected destination. The regular data-movement path discovers a pool
|
||||||
|
// from existing object state, which is ambiguous while both source and
|
||||||
|
// destination temporarily contain the version stack.
|
||||||
|
if dstPoolIdx := dataMovementDstPool(opts); dstPoolIdx != nil {
|
||||||
|
if *dstPoolIdx < 0 || *dstPoolIdx >= len(z.serverPools) {
|
||||||
|
return ObjectInfo{}, errInvalidArgument
|
||||||
|
}
|
||||||
|
if *dstPoolIdx == opts.SrcPoolIdx {
|
||||||
|
return ObjectInfo{}, DataMovementOverwriteErr{
|
||||||
|
Bucket: bucket, Object: decodeDirObject(object), VersionID: opts.VersionID,
|
||||||
|
Err: errDataMovementSrcDstPoolSame,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if z.IsSuspended(*dstPoolIdx) || z.IsPoolRebalancing(*dstPoolIdx) {
|
||||||
|
return ObjectInfo{}, toObjectErr(errDiskFull)
|
||||||
|
}
|
||||||
|
objInfo, err = z.serverPools[*dstPoolIdx].DeleteObject(ctx, bucket, object, opts)
|
||||||
|
objInfo.Name = decodeDirObject(object)
|
||||||
|
return objInfo, err
|
||||||
|
}
|
||||||
|
|
||||||
gopts := opts
|
gopts := opts
|
||||||
gopts.NoLock = true
|
gopts.NoLock = true
|
||||||
|
|
||||||
@@ -1323,10 +1378,16 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
|||||||
dstOpts.NoLock = true
|
dstOpts.NoLock = true
|
||||||
}
|
}
|
||||||
|
|
||||||
poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size)
|
poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size, dataMovementDstPool(dstOpts))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return objInfo, err
|
return objInfo, err
|
||||||
}
|
}
|
||||||
|
if dataMovementDstPool(dstOpts) != nil && poolIdx == dstOpts.SrcPoolIdx {
|
||||||
|
return ObjectInfo{}, DataMovementOverwriteErr{
|
||||||
|
Bucket: dstBucket, Object: dstObject, VersionID: dstOpts.VersionID,
|
||||||
|
Err: errDataMovementSrcDstPoolSame,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if cpSrcDstSame && srcInfo.metadataOnly {
|
if cpSrcDstSame && srcInfo.metadataOnly {
|
||||||
// Version ID is set for the destination and source == destination version ID.
|
// Version ID is set for the destination and source == destination version ID.
|
||||||
@@ -1771,29 +1832,41 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
if z.SinglePool() {
|
if z.SinglePool() {
|
||||||
return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
|
idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts))
|
||||||
}
|
|
||||||
|
|
||||||
for idx, pool := range z.serverPools {
|
|
||||||
if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// If there is a multipart upload with the same bucket/object name,
|
if dataMovementDstPool(opts) != nil && idx == opts.SrcPoolIdx {
|
||||||
// create the new multipart in the same pool, this will avoid
|
return nil, DataMovementOverwriteErr{
|
||||||
// creating two multiparts uploads in two different pools
|
Bucket: bucket, Object: object, VersionID: opts.VersionID,
|
||||||
if len(result.Uploads) != 0 {
|
Err: errDataMovementSrcDstPoolSame,
|
||||||
return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
|
}
|
||||||
|
}
|
||||||
|
return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts)
|
||||||
|
}
|
||||||
|
|
||||||
|
if dataMovementDstPool(opts) == nil {
|
||||||
|
for idx, pool := range z.serverPools {
|
||||||
|
if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// If there is a multipart upload with the same bucket/object name,
|
||||||
|
// create the new multipart in the same pool, this will avoid
|
||||||
|
// creating two multiparts uploads in two different pools.
|
||||||
|
if len(result.Uploads) != 0 {
|
||||||
|
return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// any parallel writes on the object will block for this poolIdx
|
// any parallel writes on the object will block for this poolIdx
|
||||||
// to return since this holds a read lock on the namespace.
|
// to return since this holds a read lock on the namespace.
|
||||||
idx, err := z.getPoolIdx(ctx, bucket, object, -1)
|
idx, err := z.getPoolIdx(ctx, bucket, object, -1, dataMovementDstPool(opts))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -1827,7 +1900,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
|
|||||||
}
|
}
|
||||||
|
|
||||||
if z.SinglePool() {
|
if z.SinglePool() {
|
||||||
_, err := z.getPoolIdx(ctx, bucket, object, data.Size())
|
_, err := z.getPoolIdx(ctx, bucket, object, data.Size(), dataMovementDstPool(opts))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return PartInfo{}, err
|
return PartInfo{}, err
|
||||||
}
|
}
|
||||||
@@ -2987,7 +3060,7 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje
|
|||||||
defer ns.Unlock(lkctx)
|
defer ns.Unlock(lkctx)
|
||||||
opts.NoLock = true
|
opts.NoLock = true
|
||||||
}
|
}
|
||||||
idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size)
|
idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size, dataMovementDstPool(opts))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,263 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/klauspost/compress/zstd"
|
||||||
|
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||||
|
"github.com/minio/minio/internal/config/ilm"
|
||||||
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
)
|
||||||
|
|
||||||
|
func accessLifecycleForTest(t *testing.T) *lifecycle.Lifecycle {
|
||||||
|
t.Helper()
|
||||||
|
xml := "<LifecycleConfiguration>" +
|
||||||
|
"<Rule><ID>access</ID><Status>Enabled</Status>" +
|
||||||
|
"<AccessTransition><Window>10m</Window><PromoteAfterAccesses>100</PromoteAfterAccesses>" +
|
||||||
|
"<DemoteAfterAccesses>5</DemoteAfterAccesses><DemoteAfterIdle>1h</DemoteAfterIdle></AccessTransition>" +
|
||||||
|
"</Rule></LifecycleConfiguration>"
|
||||||
|
lc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
return lc
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTierStampAndDemoteEligibility(t *testing.T) {
|
||||||
|
oldTracker := globalAccessTracker
|
||||||
|
globalAccessTracker = newAccessTracker()
|
||||||
|
t.Cleanup(func() { globalAccessTracker = oldTracker })
|
||||||
|
|
||||||
|
now := time.Now()
|
||||||
|
cfg := ilm.Config{
|
||||||
|
AccessTiering: true, AccessPools: []int{0, 1},
|
||||||
|
AccessBinWidth: time.Minute, AccessBins: 12,
|
||||||
|
AccessMinResidency: 30 * time.Minute,
|
||||||
|
}
|
||||||
|
oi := ObjectInfo{
|
||||||
|
Bucket: "bucket", Name: "object", Size: 10, IsLatest: true,
|
||||||
|
UserDefined: map[string]string{
|
||||||
|
accessTierMetadataKey: "0:" + strconv.FormatInt(now.Add(-2*time.Hour).UnixNano(), 10),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if !accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
|
||||||
|
t.Fatal("cold, resident object should be demotion eligible")
|
||||||
|
}
|
||||||
|
oi.UserDefined[accessTierMetadataKey] = "0:" + strconv.FormatInt(now.Add(-time.Minute).UnixNano(), 10)
|
||||||
|
if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
|
||||||
|
t.Fatal("object inside minimum residency was eligible")
|
||||||
|
}
|
||||||
|
oi.UserDefined[accessTierMetadataKey] = "broken"
|
||||||
|
if accessDemoteEligible(accessLifecycleForTest(t), oi, 0, cfg, now) {
|
||||||
|
t.Fatal("object with malformed marker was eligible")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTierReservationsEnforceCaps(t *testing.T) {
|
||||||
|
state := newAccessTierState(t.Context())
|
||||||
|
state.usageReady = true
|
||||||
|
state.baseUsage["a"] = 80
|
||||||
|
state.baseTotal = 180
|
||||||
|
|
||||||
|
cfg := ilm.Config{AccessMaxSize: 200}
|
||||||
|
task := accessTierTask{ctx: t.Context(), bucket: "a", object: "one", bytes: 30}
|
||||||
|
if reason, ok := state.reservePromotion(task, cfg, 0); ok || reason != "max-size" {
|
||||||
|
t.Fatalf("max-size reserve = %q/%v", reason, ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
cfg.AccessMaxSize = 0
|
||||||
|
if reason, ok := state.reservePromotion(task, cfg, 100); ok || reason != "quota" {
|
||||||
|
t.Fatalf("quota reserve = %q/%v", reason, ok)
|
||||||
|
}
|
||||||
|
|
||||||
|
task.bytes = 20
|
||||||
|
if reason, ok := state.reservePromotion(task, cfg, 100); !ok || reason != "" {
|
||||||
|
t.Fatalf("valid reserve = %q/%v", reason, ok)
|
||||||
|
}
|
||||||
|
if got := state.bucketUsageLocked("a"); got != 100 {
|
||||||
|
t.Fatalf("reserved bucket usage = %d, want 100", got)
|
||||||
|
}
|
||||||
|
state.releasePending(task, true)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDataMovementDestinationIsGated(t *testing.T) {
|
||||||
|
dst := 0
|
||||||
|
if got := dataMovementDstPool(ObjectOptions{DstPoolIdx: &dst}); got != nil {
|
||||||
|
t.Fatal("destination honored without DataMovement")
|
||||||
|
}
|
||||||
|
if got := dataMovementDstPool(ObjectOptions{DataMovement: true, DstPoolIdx: &dst}); got == nil || *got != 0 {
|
||||||
|
t.Fatalf("destination = %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestForcedDestinationPoolSelection(t *testing.T) {
|
||||||
|
z := &erasureServerPools{serverPools: make([]*erasureSets, 2)}
|
||||||
|
dst := 1
|
||||||
|
if got, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err != nil || got != dst {
|
||||||
|
t.Fatalf("destination = %d, err = %v", got, err)
|
||||||
|
}
|
||||||
|
bad := 2
|
||||||
|
if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &bad); !errors.Is(err, errInvalidArgument) {
|
||||||
|
t.Fatalf("out-of-range error = %v", err)
|
||||||
|
}
|
||||||
|
z.poolMeta.Pools = make([]PoolStatus, 2)
|
||||||
|
z.poolMeta.Pools[1].Decommission = &PoolDecommissionInfo{}
|
||||||
|
if _, err := z.getPoolIdx(context.Background(), "bucket", "object", 1, &dst); err == nil {
|
||||||
|
t.Fatal("suspended destination was accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestHotTierAccounting(t *testing.T) {
|
||||||
|
entry := dataUsageEntry{}
|
||||||
|
entry.addSizes(sizeSummary{totalSize: 100, hotTierSize: 60, versions: 1})
|
||||||
|
entry.merge(dataUsageEntry{Size: 50, HotTierSize: 25})
|
||||||
|
if entry.Size != 150 || entry.HotTierSize != 85 {
|
||||||
|
t.Fatalf("usage = size:%d hot:%d", entry.Size, entry.HotTierSize)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestScannerCyclesApart(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
current, previous uint32
|
||||||
|
want uint32
|
||||||
|
}{
|
||||||
|
{current: 12, previous: 10, want: 2},
|
||||||
|
{current: 0, previous: ^uint32(0), want: 1},
|
||||||
|
// A cycle counter reset is not evidence that a complete pass covered
|
||||||
|
// recent moves, so it must not release conservative deltas.
|
||||||
|
{current: 1, previous: 100, want: 0},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
if got := scannerCyclesApart(tt.current, tt.previous); got != tt.want {
|
||||||
|
t.Fatalf("scannerCyclesApart(%d, %d) = %d, want %d", tt.current, tt.previous, got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDataUsageCacheV8Migration(t *testing.T) {
|
||||||
|
var encoded bytes.Buffer
|
||||||
|
if err := encoded.WriteByte(dataUsageCacheVerV8); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
zw, err := zstd.NewWriter(&encoded)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
mw := msgp.NewWriter(zw)
|
||||||
|
if err = mw.WriteMapHeader(2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteString("Info"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info := dataUsageCacheInfo{Name: dataUsageRoot, NextCycle: 17, LastUpdate: time.Now().UTC()}
|
||||||
|
if err = info.EncodeMsg(mw); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteString("Cache"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteMapHeader(1); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteString("entry"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
// A v8 entry has no hts field. Encoding only populated fields also
|
||||||
|
// verifies that its map decoder retains normal msgpack compatibility.
|
||||||
|
if err = mw.WriteMapHeader(2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteString("sz"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteInt64(123); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteString("os"); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.WriteUint64(2); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = mw.Flush(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err = zw.Close(); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var migrated dataUsageCache
|
||||||
|
if err = migrated.deserialize(bytes.NewReader(encoded.Bytes())); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
entry := migrated.Cache["entry"]
|
||||||
|
if entry.Size != 123 || entry.Objects != 2 || entry.HotTierSize != 0 {
|
||||||
|
t.Fatalf("migrated entry = size:%d objects:%d hot:%d", entry.Size, entry.Objects, entry.HotTierSize)
|
||||||
|
}
|
||||||
|
if migrated.Info.NextCycle != 17 || !migrated.Info.LastUpdate.Equal(info.LastUpdate) {
|
||||||
|
t.Fatalf("migrated cache info = %+v", migrated.Info)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The stamp written on every moved version and the stamp the rollback path
|
||||||
|
// matches against must agree, otherwise rollback silently skips its own work.
|
||||||
|
func TestAccessTierStampRoundTrip(t *testing.T) {
|
||||||
|
movedAt := time.Now().UnixNano()
|
||||||
|
stamp := accessTierStamp(3, movedAt)
|
||||||
|
|
||||||
|
pool, at, ok := parseAccessTierStamp(map[string]string{accessTierMetadataKey: stamp})
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("stamp %q did not parse", stamp)
|
||||||
|
}
|
||||||
|
if pool != 3 {
|
||||||
|
t.Fatalf("pool = %d, want 3", pool)
|
||||||
|
}
|
||||||
|
if at.UnixNano() != movedAt {
|
||||||
|
t.Fatalf("movedAt = %d, want %d", at.UnixNano(), movedAt)
|
||||||
|
}
|
||||||
|
// A different move of the same object must not match, so a concurrent
|
||||||
|
// client overwrite is never mistaken for our own copy.
|
||||||
|
if stamp == accessTierStamp(3, movedAt+1) {
|
||||||
|
t.Fatal("stamps from distinct moves collided")
|
||||||
|
}
|
||||||
|
if stamp == accessTierStamp(4, movedAt) {
|
||||||
|
t.Fatal("stamps from distinct pools collided")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessHitsMightPromote(t *testing.T) {
|
||||||
|
lc := accessLifecycleForTest(t)
|
||||||
|
hot := accessEntry{Bins: []uint32{100}, HeadAt: 0}
|
||||||
|
cold := accessEntry{Bins: []uint32{1}, HeadAt: 0}
|
||||||
|
if !accessHitsMightPromote(lc, "object", hot, 60) {
|
||||||
|
t.Fatal("object above promote threshold was rejected")
|
||||||
|
}
|
||||||
|
if accessHitsMightPromote(lc, "object", cold, 60) {
|
||||||
|
t.Fatal("object below promote threshold was accepted")
|
||||||
|
}
|
||||||
|
|
||||||
|
xml := "<LifecycleConfiguration><Rule><ID>logs</ID><Status>Enabled</Status>" +
|
||||||
|
"<Filter><Prefix>logs/</Prefix></Filter>" +
|
||||||
|
"<AccessTransition><Window>10m</Window><PromoteAfterAccesses>100</PromoteAfterAccesses>" +
|
||||||
|
"<DemoteAfterAccesses>5</DemoteAfterAccesses><DemoteAfterIdle>1h</DemoteAfterIdle></AccessTransition>" +
|
||||||
|
"</Rule></LifecycleConfiguration>"
|
||||||
|
prefixed, err := lifecycle.ParseLifecycleConfig(strings.NewReader(xml))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if accessHitsMightPromote(prefixed, "data/object", hot, 60) {
|
||||||
|
t.Fatal("object outside the rule prefix was accepted")
|
||||||
|
}
|
||||||
|
if !accessHitsMightPromote(prefixed, "logs/object", hot, 60) {
|
||||||
|
t.Fatal("object inside the rule prefix was rejected")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,598 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"cmp"
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"slices"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/config/ilm"
|
||||||
|
"github.com/zeebo/xxh3"
|
||||||
|
)
|
||||||
|
|
||||||
|
//go:generate msgp -file=$GOFILE -unexported
|
||||||
|
//msgp:ignore accessTracker mergedAccess
|
||||||
|
|
||||||
|
const (
|
||||||
|
// accessTrackerPrefix is where each node publishes its own counters.
|
||||||
|
// One object per node, merged by every node on a timer.
|
||||||
|
accessTrackerPrefix = minioConfigPrefix + "/ilm/access"
|
||||||
|
|
||||||
|
// accessQueueSize bounds the GET -> tracker handoff. Overflow drops
|
||||||
|
// samples rather than slowing down reads.
|
||||||
|
accessQueueSize = 100000
|
||||||
|
|
||||||
|
// accessMaxDemoteCandidates bounds what the scanner may hand over in a
|
||||||
|
// single flush interval.
|
||||||
|
accessMaxDemoteCandidates = 100000
|
||||||
|
|
||||||
|
// accessShardStaleFactor multiplies the flush interval to decide when a
|
||||||
|
// peer's counters are too old to trust, e.g. after a node is removed.
|
||||||
|
accessShardStaleFactor = 5
|
||||||
|
)
|
||||||
|
|
||||||
|
func accessShardFresh(now int64, shard accessShard, stale int64) bool {
|
||||||
|
if shard.UpdatedAt <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if stale <= 0 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
age := now - shard.UpdatedAt
|
||||||
|
return age >= -stale && age <= stale
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessEntry is a rolling hit counter for one object. Bins[0] is the current
|
||||||
|
// bin and each subsequent bin is one bin-width older, so a rule asking for
|
||||||
|
// "100 hits in 10 minutes" sums the newest ceil(10m/binWidth) bins.
|
||||||
|
//
|
||||||
|
// A fixed window is used rather than an exponentially decayed score because
|
||||||
|
// the rule is stated to operators in exactly those terms.
|
||||||
|
type accessEntry struct {
|
||||||
|
Bins []uint32 `msg:"b"`
|
||||||
|
HeadAt int64 `msg:"h"` // unix seconds at the start of Bins[0]
|
||||||
|
LastAt int64 `msg:"l"` // unix seconds of the most recent hit
|
||||||
|
}
|
||||||
|
|
||||||
|
// demoteCandidate is an object the scanner found sitting on a fast pool with
|
||||||
|
// no recent reads. Candidates ride the node's own counter shard so they reach
|
||||||
|
// the leader without a new peer RPC.
|
||||||
|
type demoteCandidate struct {
|
||||||
|
Bucket string `msg:"b"`
|
||||||
|
Object string `msg:"o"`
|
||||||
|
Pool int `msg:"p"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessShard is what one node publishes. BinWidth is carried so a peer that
|
||||||
|
// has not yet picked up a configuration change is ignored rather than merged
|
||||||
|
// with mismatched bins.
|
||||||
|
type accessShard struct {
|
||||||
|
UpdatedAt int64 `msg:"u"`
|
||||||
|
BinWidth int64 `msg:"bw"`
|
||||||
|
Entries map[string]accessEntry `msg:"e"`
|
||||||
|
Demote []demoteCandidate `msg:"d"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// binStart truncates a unix timestamp to the start of its bin.
|
||||||
|
func binStart(now, binWidth int64) int64 {
|
||||||
|
if binWidth <= 0 {
|
||||||
|
return now
|
||||||
|
}
|
||||||
|
return now - now%binWidth
|
||||||
|
}
|
||||||
|
|
||||||
|
// rollTo advances the counter to now, zeroing the bins that elapsed since the
|
||||||
|
// last update and resizing if the configured bin count changed.
|
||||||
|
func (e *accessEntry) rollTo(now, binWidth int64, nbins int) {
|
||||||
|
if nbins <= 0 || binWidth <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(e.Bins) != nbins {
|
||||||
|
resized := make([]uint32, nbins)
|
||||||
|
copy(resized, e.Bins)
|
||||||
|
e.Bins = resized
|
||||||
|
}
|
||||||
|
head := binStart(now, binWidth)
|
||||||
|
if e.HeadAt == 0 {
|
||||||
|
e.HeadAt = head
|
||||||
|
return
|
||||||
|
}
|
||||||
|
steps := (head - e.HeadAt) / binWidth
|
||||||
|
if steps <= 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if steps >= int64(nbins) {
|
||||||
|
clear(e.Bins)
|
||||||
|
} else {
|
||||||
|
copy(e.Bins[steps:], e.Bins[:nbins-int(steps)])
|
||||||
|
clear(e.Bins[:steps])
|
||||||
|
}
|
||||||
|
e.HeadAt = head
|
||||||
|
}
|
||||||
|
|
||||||
|
// hits returns the number of accesses recorded over the newest bins covering
|
||||||
|
// window. A window longer than the configured history is clamped to it.
|
||||||
|
//
|
||||||
|
// The newest bin is partial, so the covered span is between window-binWidth
|
||||||
|
// and window. Operators tune resolution with ilm access_bin_width.
|
||||||
|
func (e accessEntry) hits(window time.Duration, binWidth int64) uint64 {
|
||||||
|
if binWidth <= 0 || len(e.Bins) == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
n := int((int64(window/time.Second) + binWidth - 1) / binWidth)
|
||||||
|
if n < 1 {
|
||||||
|
n = 1
|
||||||
|
}
|
||||||
|
if n > len(e.Bins) {
|
||||||
|
n = len(e.Bins)
|
||||||
|
}
|
||||||
|
var total uint64
|
||||||
|
for _, v := range e.Bins[:n] {
|
||||||
|
total += uint64(v)
|
||||||
|
}
|
||||||
|
return total
|
||||||
|
}
|
||||||
|
|
||||||
|
// total is the whole retained history, used to decide what to evict.
|
||||||
|
func (e accessEntry) total() uint64 {
|
||||||
|
var t uint64
|
||||||
|
for _, v := range e.Bins {
|
||||||
|
t += uint64(v)
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeFrom adds another node's counters for the same object. Both sides must
|
||||||
|
// already be rolled to the same head.
|
||||||
|
func (e *accessEntry) mergeFrom(o accessEntry) {
|
||||||
|
for i := range e.Bins {
|
||||||
|
if i < len(o.Bins) {
|
||||||
|
total := uint64(e.Bins[i]) + uint64(o.Bins[i])
|
||||||
|
if total > uint64(^uint32(0)) {
|
||||||
|
total = uint64(^uint32(0))
|
||||||
|
}
|
||||||
|
e.Bins[i] = uint32(total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if o.LastAt > e.LastAt {
|
||||||
|
e.LastAt = o.LastAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergedAccess is an immutable cluster-wide snapshot. Readers take it from an
|
||||||
|
// atomic pointer, so the hot scanner and sweep paths never take a lock.
|
||||||
|
type mergedAccess struct {
|
||||||
|
entries map[string]accessEntry
|
||||||
|
binWidth int64
|
||||||
|
at int64
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mergedAccess) hits(key string, window time.Duration) uint64 {
|
||||||
|
if m == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
e, ok := m.entries[key]
|
||||||
|
if !ok {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return e.hits(window, m.binWidth)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (m *mergedAccess) lastAccess(key string) int64 {
|
||||||
|
if m == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return m.entries[key].LastAt
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessTracker records how often each object is read.
|
||||||
|
//
|
||||||
|
// Ownership is deliberately narrow: the live counter map is touched only by
|
||||||
|
// run()'s goroutine, so it needs no lock. Everything read from elsewhere goes
|
||||||
|
// through the immutable merged snapshot.
|
||||||
|
type accessTracker struct {
|
||||||
|
ch chan string
|
||||||
|
enabled atomic.Bool
|
||||||
|
merged atomic.Pointer[mergedAccess]
|
||||||
|
|
||||||
|
// Demote candidates arrive from scanner goroutines, so this one does
|
||||||
|
// need a lock. It is small: only objects we previously promoted.
|
||||||
|
demoteMu sync.Mutex
|
||||||
|
demote map[string]demoteCandidate
|
||||||
|
|
||||||
|
dropped atomic.Uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
var globalAccessTracker = newAccessTracker()
|
||||||
|
|
||||||
|
func newAccessTracker() *accessTracker {
|
||||||
|
return &accessTracker{
|
||||||
|
ch: make(chan string, accessQueueSize),
|
||||||
|
demote: make(map[string]demoteCandidate),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessKey is the tracker's map key. Bucket names cannot contain '/', so the
|
||||||
|
// join is unambiguous.
|
||||||
|
func accessKey(bucket, object string) string {
|
||||||
|
return bucket + "/" + object
|
||||||
|
}
|
||||||
|
|
||||||
|
func splitAccessKey(key string) (bucket, object string, ok bool) {
|
||||||
|
bucket, object, ok = strings.Cut(key, "/")
|
||||||
|
if !ok || bucket == "" || object == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
return bucket, object, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// note records one read. It is called from the GET path and must never block
|
||||||
|
// or allocate meaningfully: on a full queue the sample is dropped.
|
||||||
|
func (t *accessTracker) note(bucket, object string) {
|
||||||
|
if t == nil || !t.enabled.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case t.ch <- accessKey(bucket, object):
|
||||||
|
default:
|
||||||
|
t.dropped.Add(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// noteDemoteCandidate is called by the scanner for an object it found on a
|
||||||
|
// fast pool that has gone quiet. The leader picks these up on the next merge.
|
||||||
|
func (t *accessTracker) noteDemoteCandidate(bucket, object string, pool int) {
|
||||||
|
if t == nil || !t.enabled.Load() {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.demoteMu.Lock()
|
||||||
|
defer t.demoteMu.Unlock()
|
||||||
|
if len(t.demote) >= accessMaxDemoteCandidates {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.demote[accessKey(bucket, object)] = demoteCandidate{Bucket: bucket, Object: object, Pool: pool}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hits reports cluster-wide accesses to an object over window.
|
||||||
|
func (t *accessTracker) hits(bucket, object string, window time.Duration) uint64 {
|
||||||
|
if t == nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return t.merged.Load().hits(accessKey(bucket, object), window)
|
||||||
|
}
|
||||||
|
|
||||||
|
// lastAccess reports the cluster-wide time an object was last read. A zero
|
||||||
|
// time means "no read on record", which for demotion purposes is idle.
|
||||||
|
func (t *accessTracker) lastAccess(bucket, object string) time.Time {
|
||||||
|
if t == nil {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
sec := t.merged.Load().lastAccess(accessKey(bucket, object))
|
||||||
|
if sec == 0 {
|
||||||
|
return time.Time{}
|
||||||
|
}
|
||||||
|
return time.Unix(sec, 0)
|
||||||
|
}
|
||||||
|
|
||||||
|
// snapshot returns the current merged view, or nil if none has been published.
|
||||||
|
func (t *accessTracker) snapshot() *mergedAccess {
|
||||||
|
if t == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return t.merged.Load()
|
||||||
|
}
|
||||||
|
|
||||||
|
// takeDemoteCandidates drains and returns the pending candidates.
|
||||||
|
func (t *accessTracker) takeDemoteCandidates() []demoteCandidate {
|
||||||
|
t.demoteMu.Lock()
|
||||||
|
defer t.demoteMu.Unlock()
|
||||||
|
if len(t.demote) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
out := make([]demoteCandidate, 0, len(t.demote))
|
||||||
|
for _, c := range t.demote {
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
t.demote = make(map[string]demoteCandidate)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// restoreDemoteCandidates puts candidates back when the publisher is still
|
||||||
|
// busy. Dropping access samples is acceptable; dropping the only scanner
|
||||||
|
// discovery of an idle promoted object would delay demotion by a full scan.
|
||||||
|
func (t *accessTracker) restoreDemoteCandidates(candidates []demoteCandidate) {
|
||||||
|
if len(candidates) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.demoteMu.Lock()
|
||||||
|
defer t.demoteMu.Unlock()
|
||||||
|
for _, c := range candidates {
|
||||||
|
if len(t.demote) >= accessMaxDemoteCandidates {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
t.demote[accessKey(c.Bucket, c.Object)] = c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// shardName is this node's counter object. The node name is hashed so that
|
||||||
|
// host:port never has to be escaped into an object key.
|
||||||
|
func (t *accessTracker) shardName() string {
|
||||||
|
return fmt.Sprintf("%s/%016x.bin", accessTrackerPrefix, xxh3.HashString(globalLocalNodeName))
|
||||||
|
}
|
||||||
|
|
||||||
|
// run owns the live counter map. It drains reads, and on every flush interval
|
||||||
|
// hands a marshaled shard to a background publisher.
|
||||||
|
//
|
||||||
|
// Everything here is best effort: this is accounting for a background data
|
||||||
|
// movement decision, not a durability path.
|
||||||
|
func (t *accessTracker) run(ctx context.Context, objAPI ObjectLayer) {
|
||||||
|
cfg := globalILMConfig.accessCfg()
|
||||||
|
t.enabled.Store(cfg.AccessTiering)
|
||||||
|
|
||||||
|
live := make(map[string]accessEntry)
|
||||||
|
if cfg.AccessTiering {
|
||||||
|
if buf, err := readConfig(ctx, objAPI, t.shardName()); err == nil {
|
||||||
|
var previous accessShard
|
||||||
|
if _, err = previous.UnmarshalMsg(buf); err == nil && previous.BinWidth == int64(cfg.AccessBinWidth/time.Second) {
|
||||||
|
now := time.Now().Unix()
|
||||||
|
for key, entry := range previous.Entries {
|
||||||
|
entry.rollTo(now, previous.BinWidth, cfg.AccessBins)
|
||||||
|
if entry.total() != 0 {
|
||||||
|
live[key] = entry
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
ticker := time.NewTicker(cfg.AccessFlush)
|
||||||
|
defer ticker.Stop()
|
||||||
|
|
||||||
|
// One publisher goroutine keeps object-layer I/O off the drain loop.
|
||||||
|
type pub struct {
|
||||||
|
shard accessShard
|
||||||
|
cfg ilm.Config
|
||||||
|
}
|
||||||
|
pubCh := make(chan pub, 1)
|
||||||
|
go func() {
|
||||||
|
for p := range pubCh {
|
||||||
|
t.publish(ctx, objAPI, p.shard, p.cfg)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
defer close(pubCh)
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return
|
||||||
|
|
||||||
|
case key := <-t.ch:
|
||||||
|
now := time.Now().Unix()
|
||||||
|
e := live[key]
|
||||||
|
e.rollTo(now, int64(cfg.AccessBinWidth/time.Second), cfg.AccessBins)
|
||||||
|
if len(e.Bins) > 0 {
|
||||||
|
if e.Bins[0] != ^uint32(0) {
|
||||||
|
e.Bins[0]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
e.LastAt = now
|
||||||
|
live[key] = e
|
||||||
|
|
||||||
|
case <-ticker.C:
|
||||||
|
newCfg := globalILMConfig.accessCfg()
|
||||||
|
t.enabled.Store(newCfg.AccessTiering)
|
||||||
|
if newCfg.AccessFlush != cfg.AccessFlush && newCfg.AccessFlush > 0 {
|
||||||
|
ticker.Reset(newCfg.AccessFlush)
|
||||||
|
}
|
||||||
|
cfg = newCfg
|
||||||
|
if !cfg.AccessTiering {
|
||||||
|
// Feature turned off: release the counters rather than
|
||||||
|
// holding a stale working set for the process lifetime.
|
||||||
|
clear(live)
|
||||||
|
t.merged.Store(nil)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now().Unix()
|
||||||
|
binWidth := int64(cfg.AccessBinWidth / time.Second)
|
||||||
|
t.evict(live, now, binWidth, cfg)
|
||||||
|
|
||||||
|
shard := accessShard{
|
||||||
|
UpdatedAt: now,
|
||||||
|
BinWidth: binWidth,
|
||||||
|
Entries: make(map[string]accessEntry, len(live)),
|
||||||
|
Demote: t.takeDemoteCandidates(),
|
||||||
|
}
|
||||||
|
for k, e := range live {
|
||||||
|
e.Bins = slices.Clone(e.Bins)
|
||||||
|
shard.Entries[k] = e
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case pubCh <- pub{shard: shard, cfg: cfg}:
|
||||||
|
default:
|
||||||
|
// Previous publish still running; skip this round
|
||||||
|
// rather than queueing work we cannot keep up with.
|
||||||
|
t.restoreDemoteCandidates(shard.Demote)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// evict rolls every counter forward and drops the ones with no hits left in
|
||||||
|
// the retained history, then enforces the tracked-object cap.
|
||||||
|
//
|
||||||
|
// ponytail: amortized O(n) sweep on the flush tick; a heap would only pay off
|
||||||
|
// past ~10M tracked keys.
|
||||||
|
func (t *accessTracker) evict(live map[string]accessEntry, now, binWidth int64, cfg ilm.Config) {
|
||||||
|
for k, e := range live {
|
||||||
|
e.rollTo(now, binWidth, cfg.AccessBins)
|
||||||
|
if e.total() == 0 {
|
||||||
|
delete(live, k)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
live[k] = e
|
||||||
|
}
|
||||||
|
if len(live) <= cfg.AccessMaxTracked {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Over the cap: keep the hottest and drop the rest. Objects that fall
|
||||||
|
// out are cold by construction, which is exactly what the demotion path
|
||||||
|
// already assumes about anything missing from the map.
|
||||||
|
type kt struct {
|
||||||
|
key string
|
||||||
|
total uint64
|
||||||
|
}
|
||||||
|
all := make([]kt, 0, len(live))
|
||||||
|
for k, e := range live {
|
||||||
|
all = append(all, kt{k, e.total()})
|
||||||
|
}
|
||||||
|
slices.SortFunc(all, func(a, b kt) int { return cmp.Compare(b.total, a.total) })
|
||||||
|
for _, x := range all[cfg.AccessMaxTracked:] {
|
||||||
|
delete(live, x.key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// publish writes this node's shard and rebuilds the merged snapshot from every
|
||||||
|
// node's shard. Both halves are best effort.
|
||||||
|
func (t *accessTracker) publish(ctx context.Context, objAPI ObjectLayer, shard accessShard, cfg ilm.Config) {
|
||||||
|
buf, err := shard.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
ilmLogIf(ctx, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := saveConfig(ctx, objAPI, t.shardName(), buf); err != nil {
|
||||||
|
ilmLogIf(ctx, err)
|
||||||
|
// Still rebuild the snapshot below: our own counters are already
|
||||||
|
// in hand and a stale peer view beats no view.
|
||||||
|
}
|
||||||
|
t.merged.Store(t.mergeShards(ctx, objAPI, shard, cfg))
|
||||||
|
}
|
||||||
|
|
||||||
|
// mergeShards sums every live node's counters, including our own in-memory
|
||||||
|
// shard so this node's most recent reads are never a flush behind.
|
||||||
|
func (t *accessTracker) mergeShards(ctx context.Context, objAPI ObjectLayer, own accessShard, cfg ilm.Config) *mergedAccess {
|
||||||
|
now := time.Now().Unix()
|
||||||
|
binWidth := int64(cfg.AccessBinWidth / time.Second)
|
||||||
|
out := &mergedAccess{
|
||||||
|
entries: make(map[string]accessEntry, len(own.Entries)),
|
||||||
|
binWidth: binWidth,
|
||||||
|
at: now,
|
||||||
|
}
|
||||||
|
|
||||||
|
add := func(s accessShard) {
|
||||||
|
if s.BinWidth != binWidth {
|
||||||
|
// A peer has not yet picked up a bin-width change; merging
|
||||||
|
// its bins would silently mis-scale the counts.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for k, e := range s.Entries {
|
||||||
|
e.rollTo(now, binWidth, cfg.AccessBins)
|
||||||
|
cur, ok := out.entries[k]
|
||||||
|
if !ok {
|
||||||
|
cur = accessEntry{Bins: make([]uint32, cfg.AccessBins)}
|
||||||
|
}
|
||||||
|
cur.mergeFrom(e)
|
||||||
|
out.entries[k] = cur
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
add(own)
|
||||||
|
|
||||||
|
ownName := t.shardName()
|
||||||
|
stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor
|
||||||
|
for _, name := range t.listShards(ctx, objAPI) {
|
||||||
|
if name == ownName {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
buf, err := readConfig(ctx, objAPI, name)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var s accessShard
|
||||||
|
if _, err := s.UnmarshalMsg(buf); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !accessShardFresh(now, s, stale) {
|
||||||
|
continue // node is gone or wedged
|
||||||
|
}
|
||||||
|
add(s)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func (t *accessTracker) listShards(ctx context.Context, objAPI ObjectLayer) []string {
|
||||||
|
res, err := objAPI.ListObjects(ctx, minioMetaBucket, accessTrackerPrefix+"/", "", "", maxObjectList)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
names := make([]string, 0, len(res.Objects))
|
||||||
|
for _, o := range res.Objects {
|
||||||
|
if strings.HasSuffix(o.Name, ".bin") {
|
||||||
|
names = append(names, o.Name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return names
|
||||||
|
}
|
||||||
|
|
||||||
|
// collectDemoteCandidates returns every node's pending demote candidates. Only
|
||||||
|
// the leader calls this, right before running a demotion pass.
|
||||||
|
//
|
||||||
|
// Our own shard is read back rather than skipped: run() drains the local map
|
||||||
|
// into the published shard, so anything found since the last leader pass lives
|
||||||
|
// there, not in memory. The local map is still drained here to pick up
|
||||||
|
// candidates recorded since that publish.
|
||||||
|
func (t *accessTracker) collectDemoteCandidates(ctx context.Context, objAPI ObjectLayer, cfg ilm.Config) []demoteCandidate {
|
||||||
|
seen := make(map[string]struct{})
|
||||||
|
var out []demoteCandidate
|
||||||
|
|
||||||
|
for _, c := range t.takeDemoteCandidates() {
|
||||||
|
key := accessKey(c.Bucket, c.Object)
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
|
||||||
|
now := time.Now().Unix()
|
||||||
|
stale := int64(cfg.AccessFlush/time.Second) * accessShardStaleFactor
|
||||||
|
for _, name := range t.listShards(ctx, objAPI) {
|
||||||
|
buf, err := readConfig(ctx, objAPI, name)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
var s accessShard
|
||||||
|
if _, err := s.UnmarshalMsg(buf); err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !accessShardFresh(now, s, stale) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for _, c := range s.Demote {
|
||||||
|
key := accessKey(c.Bucket, c.Object)
|
||||||
|
if _, dup := seen[key]; dup {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
seen[key] = struct{}{}
|
||||||
|
out = append(out, c)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,742 @@
|
|||||||
|
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *accessEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "b":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, err = dc.ReadArrayHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bins")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Bins) >= int(zb0002) {
|
||||||
|
z.Bins = (z.Bins)[:zb0002]
|
||||||
|
} else {
|
||||||
|
z.Bins = make([]uint32, zb0002)
|
||||||
|
}
|
||||||
|
for za0001 := range z.Bins {
|
||||||
|
z.Bins[za0001], err = dc.ReadUint32()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bins", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "h":
|
||||||
|
z.HeadAt, err = dc.ReadInt64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "HeadAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "l":
|
||||||
|
z.LastAt, err = dc.ReadInt64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "LastAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeMsg implements msgp.Encodable
|
||||||
|
func (z *accessEntry) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
|
// map header, size 3
|
||||||
|
// write "b"
|
||||||
|
err = en.Append(0x83, 0xa1, 0x62)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteArrayHeader(uint32(len(z.Bins)))
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bins")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0001 := range z.Bins {
|
||||||
|
err = en.WriteUint32(z.Bins[za0001])
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bins", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// write "h"
|
||||||
|
err = en.Append(0xa1, 0x68)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt64(z.HeadAt)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "HeadAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "l"
|
||||||
|
err = en.Append(0xa1, 0x6c)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt64(z.LastAt)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "LastAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalMsg implements msgp.Marshaler
|
||||||
|
func (z *accessEntry) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
|
o = msgp.Require(b, z.Msgsize())
|
||||||
|
// map header, size 3
|
||||||
|
// string "b"
|
||||||
|
o = append(o, 0x83, 0xa1, 0x62)
|
||||||
|
o = msgp.AppendArrayHeader(o, uint32(len(z.Bins)))
|
||||||
|
for za0001 := range z.Bins {
|
||||||
|
o = msgp.AppendUint32(o, z.Bins[za0001])
|
||||||
|
}
|
||||||
|
// string "h"
|
||||||
|
o = append(o, 0xa1, 0x68)
|
||||||
|
o = msgp.AppendInt64(o, z.HeadAt)
|
||||||
|
// string "l"
|
||||||
|
o = append(o, 0xa1, 0x6c)
|
||||||
|
o = msgp.AppendInt64(o, z.LastAt)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *accessEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "b":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bins")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Bins) >= int(zb0002) {
|
||||||
|
z.Bins = (z.Bins)[:zb0002]
|
||||||
|
} else {
|
||||||
|
z.Bins = make([]uint32, zb0002)
|
||||||
|
}
|
||||||
|
for za0001 := range z.Bins {
|
||||||
|
z.Bins[za0001], bts, err = msgp.ReadUint32Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bins", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case "h":
|
||||||
|
z.HeadAt, bts, err = msgp.ReadInt64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "HeadAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "l":
|
||||||
|
z.LastAt, bts, err = msgp.ReadInt64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "LastAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z *accessEntry) Msgsize() (s int) {
|
||||||
|
s = 1 + 2 + msgp.ArrayHeaderSize + (len(z.Bins) * (msgp.Uint32Size)) + 2 + msgp.Int64Size + 2 + msgp.Int64Size
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *accessShard) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "u":
|
||||||
|
z.UpdatedAt, err = dc.ReadInt64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "UpdatedAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "bw":
|
||||||
|
z.BinWidth, err = dc.ReadInt64()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "BinWidth")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "e":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if z.Entries == nil {
|
||||||
|
z.Entries = make(map[string]accessEntry, zb0002)
|
||||||
|
} else if len(z.Entries) > 0 {
|
||||||
|
clear(z.Entries)
|
||||||
|
}
|
||||||
|
for zb0002 > 0 {
|
||||||
|
zb0002--
|
||||||
|
var za0001 string
|
||||||
|
za0001, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
var za0002 accessEntry
|
||||||
|
err = za0002.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.Entries[za0001] = za0002
|
||||||
|
}
|
||||||
|
case "d":
|
||||||
|
var zb0003 uint32
|
||||||
|
zb0003, err = dc.ReadArrayHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Demote) >= int(zb0003) {
|
||||||
|
z.Demote = (z.Demote)[:zb0003]
|
||||||
|
} else {
|
||||||
|
z.Demote = make([]demoteCandidate, zb0003)
|
||||||
|
}
|
||||||
|
for za0003 := range z.Demote {
|
||||||
|
var zb0004 uint32
|
||||||
|
zb0004, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0004 > 0 {
|
||||||
|
zb0004--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "b":
|
||||||
|
z.Demote[za0003].Bucket, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "o":
|
||||||
|
z.Demote[za0003].Object, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Object")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "p":
|
||||||
|
z.Demote[za0003].Pool, err = dc.ReadInt()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Pool")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeMsg implements msgp.Encodable
|
||||||
|
func (z *accessShard) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
|
// map header, size 4
|
||||||
|
// write "u"
|
||||||
|
err = en.Append(0x84, 0xa1, 0x75)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt64(z.UpdatedAt)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "UpdatedAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "bw"
|
||||||
|
err = en.Append(0xa2, 0x62, 0x77)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt64(z.BinWidth)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "BinWidth")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "e"
|
||||||
|
err = en.Append(0xa1, 0x65)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteMapHeader(uint32(len(z.Entries)))
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0001, za0002 := range z.Entries {
|
||||||
|
err = en.WriteString(za0001)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = za0002.EncodeMsg(en)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// write "d"
|
||||||
|
err = en.Append(0xa1, 0x64)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteArrayHeader(uint32(len(z.Demote)))
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for za0003 := range z.Demote {
|
||||||
|
// map header, size 3
|
||||||
|
// write "b"
|
||||||
|
err = en.Append(0x83, 0xa1, 0x62)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Demote[za0003].Bucket)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "o"
|
||||||
|
err = en.Append(0xa1, 0x6f)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Demote[za0003].Object)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Object")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "p"
|
||||||
|
err = en.Append(0xa1, 0x70)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt(z.Demote[za0003].Pool)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Pool")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalMsg implements msgp.Marshaler
|
||||||
|
func (z *accessShard) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
|
o = msgp.Require(b, z.Msgsize())
|
||||||
|
// map header, size 4
|
||||||
|
// string "u"
|
||||||
|
o = append(o, 0x84, 0xa1, 0x75)
|
||||||
|
o = msgp.AppendInt64(o, z.UpdatedAt)
|
||||||
|
// string "bw"
|
||||||
|
o = append(o, 0xa2, 0x62, 0x77)
|
||||||
|
o = msgp.AppendInt64(o, z.BinWidth)
|
||||||
|
// string "e"
|
||||||
|
o = append(o, 0xa1, 0x65)
|
||||||
|
o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
|
||||||
|
for za0001, za0002 := range z.Entries {
|
||||||
|
o = msgp.AppendString(o, za0001)
|
||||||
|
o, err = za0002.MarshalMsg(o)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// string "d"
|
||||||
|
o = append(o, 0xa1, 0x64)
|
||||||
|
o = msgp.AppendArrayHeader(o, uint32(len(z.Demote)))
|
||||||
|
for za0003 := range z.Demote {
|
||||||
|
// map header, size 3
|
||||||
|
// string "b"
|
||||||
|
o = append(o, 0x83, 0xa1, 0x62)
|
||||||
|
o = msgp.AppendString(o, z.Demote[za0003].Bucket)
|
||||||
|
// string "o"
|
||||||
|
o = append(o, 0xa1, 0x6f)
|
||||||
|
o = msgp.AppendString(o, z.Demote[za0003].Object)
|
||||||
|
// string "p"
|
||||||
|
o = append(o, 0xa1, 0x70)
|
||||||
|
o = msgp.AppendInt(o, z.Demote[za0003].Pool)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *accessShard) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "u":
|
||||||
|
z.UpdatedAt, bts, err = msgp.ReadInt64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "UpdatedAt")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "bw":
|
||||||
|
z.BinWidth, bts, err = msgp.ReadInt64Bytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "BinWidth")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "e":
|
||||||
|
var zb0002 uint32
|
||||||
|
zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if z.Entries == nil {
|
||||||
|
z.Entries = make(map[string]accessEntry, zb0002)
|
||||||
|
} else if len(z.Entries) > 0 {
|
||||||
|
clear(z.Entries)
|
||||||
|
}
|
||||||
|
for zb0002 > 0 {
|
||||||
|
var za0002 accessEntry
|
||||||
|
zb0002--
|
||||||
|
var za0001 string
|
||||||
|
za0001, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
bts, err = za0002.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Entries", za0001)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
z.Entries[za0001] = za0002
|
||||||
|
}
|
||||||
|
case "d":
|
||||||
|
var zb0003 uint32
|
||||||
|
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if cap(z.Demote) >= int(zb0003) {
|
||||||
|
z.Demote = (z.Demote)[:zb0003]
|
||||||
|
} else {
|
||||||
|
z.Demote = make([]demoteCandidate, zb0003)
|
||||||
|
}
|
||||||
|
for za0003 := range z.Demote {
|
||||||
|
var zb0004 uint32
|
||||||
|
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0004 > 0 {
|
||||||
|
zb0004--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "b":
|
||||||
|
z.Demote[za0003].Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "o":
|
||||||
|
z.Demote[za0003].Object, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Object")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "p":
|
||||||
|
z.Demote[za0003].Pool, bts, err = msgp.ReadIntBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003, "Pool")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Demote", za0003)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z *accessShard) Msgsize() (s int) {
|
||||||
|
s = 1 + 2 + msgp.Int64Size + 3 + msgp.Int64Size + 2 + msgp.MapHeaderSize
|
||||||
|
if z.Entries != nil {
|
||||||
|
for za0001, za0002 := range z.Entries {
|
||||||
|
_ = za0002
|
||||||
|
s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
s += 2 + msgp.ArrayHeaderSize
|
||||||
|
for za0003 := range z.Demote {
|
||||||
|
s += 1 + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Bucket) + 2 + msgp.StringPrefixSize + len(z.Demote[za0003].Object) + 2 + msgp.IntSize
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// DecodeMsg implements msgp.Decodable
|
||||||
|
func (z *demoteCandidate) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, err = dc.ReadMapHeader()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, err = dc.ReadMapKeyPtr()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "b":
|
||||||
|
z.Bucket, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "o":
|
||||||
|
z.Object, err = dc.ReadString()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Object")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "p":
|
||||||
|
z.Pool, err = dc.ReadInt()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Pool")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
err = dc.Skip()
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// EncodeMsg implements msgp.Encodable
|
||||||
|
func (z demoteCandidate) EncodeMsg(en *msgp.Writer) (err error) {
|
||||||
|
// map header, size 3
|
||||||
|
// write "b"
|
||||||
|
err = en.Append(0x83, 0xa1, 0x62)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Bucket)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "o"
|
||||||
|
err = en.Append(0xa1, 0x6f)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteString(z.Object)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Object")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// write "p"
|
||||||
|
err = en.Append(0xa1, 0x70)
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
err = en.WriteInt(z.Pool)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Pool")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalMsg implements msgp.Marshaler
|
||||||
|
func (z demoteCandidate) MarshalMsg(b []byte) (o []byte, err error) {
|
||||||
|
o = msgp.Require(b, z.Msgsize())
|
||||||
|
// map header, size 3
|
||||||
|
// string "b"
|
||||||
|
o = append(o, 0x83, 0xa1, 0x62)
|
||||||
|
o = msgp.AppendString(o, z.Bucket)
|
||||||
|
// string "o"
|
||||||
|
o = append(o, 0xa1, 0x6f)
|
||||||
|
o = msgp.AppendString(o, z.Object)
|
||||||
|
// string "p"
|
||||||
|
o = append(o, 0xa1, 0x70)
|
||||||
|
o = msgp.AppendInt(o, z.Pool)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalMsg implements msgp.Unmarshaler
|
||||||
|
func (z *demoteCandidate) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||||
|
var field []byte
|
||||||
|
_ = field
|
||||||
|
var zb0001 uint32
|
||||||
|
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for zb0001 > 0 {
|
||||||
|
zb0001--
|
||||||
|
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch msgp.UnsafeString(field) {
|
||||||
|
case "b":
|
||||||
|
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Bucket")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "o":
|
||||||
|
z.Object, bts, err = msgp.ReadStringBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Object")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
case "p":
|
||||||
|
z.Pool, bts, err = msgp.ReadIntBytes(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err, "Pool")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
bts, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
err = msgp.WrapError(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
o = bts
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||||
|
func (z demoteCandidate) Msgsize() (s int) {
|
||||||
|
s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object) + 2 + msgp.IntSize
|
||||||
|
return
|
||||||
|
}
|
||||||
@@ -0,0 +1,349 @@
|
|||||||
|
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/tinylib/msgp/msgp"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalaccessEntry(t *testing.T) {
|
||||||
|
v := accessEntry{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgaccessEntry(b *testing.B) {
|
||||||
|
v := accessEntry{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgaccessEntry(b *testing.B) {
|
||||||
|
v := accessEntry{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalaccessEntry(b *testing.B) {
|
||||||
|
v := accessEntry{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeaccessEntry(t *testing.T) {
|
||||||
|
v := accessEntry{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeaccessEntry Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := accessEntry{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeaccessEntry(b *testing.B) {
|
||||||
|
v := accessEntry{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeaccessEntry(b *testing.B) {
|
||||||
|
v := accessEntry{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshalaccessShard(t *testing.T) {
|
||||||
|
v := accessShard{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgaccessShard(b *testing.B) {
|
||||||
|
v := accessShard{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgaccessShard(b *testing.B) {
|
||||||
|
v := accessShard{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshalaccessShard(b *testing.B) {
|
||||||
|
v := accessShard{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodeaccessShard(t *testing.T) {
|
||||||
|
v := accessShard{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodeaccessShard Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := accessShard{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodeaccessShard(b *testing.B) {
|
||||||
|
v := accessShard{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodeaccessShard(b *testing.B) {
|
||||||
|
v := accessShard{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMarshalUnmarshaldemoteCandidate(t *testing.T) {
|
||||||
|
v := demoteCandidate{}
|
||||||
|
bts, err := v.MarshalMsg(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
left, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
|
||||||
|
left, err = msgp.Skip(bts)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(left) > 0 {
|
||||||
|
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkMarshalMsgdemoteCandidate(b *testing.B) {
|
||||||
|
v := demoteCandidate{}
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.MarshalMsg(nil)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkAppendMsgdemoteCandidate(b *testing.B) {
|
||||||
|
v := demoteCandidate{}
|
||||||
|
bts := make([]byte, 0, v.Msgsize())
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
bts, _ = v.MarshalMsg(bts[0:0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkUnmarshaldemoteCandidate(b *testing.B) {
|
||||||
|
v := demoteCandidate{}
|
||||||
|
bts, _ := v.MarshalMsg(nil)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.SetBytes(int64(len(bts)))
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
_, err := v.UnmarshalMsg(bts)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEncodeDecodedemoteCandidate(t *testing.T) {
|
||||||
|
v := demoteCandidate{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
|
||||||
|
m := v.Msgsize()
|
||||||
|
if buf.Len() > m {
|
||||||
|
t.Log("WARNING: TestEncodeDecodedemoteCandidate Msgsize() is inaccurate")
|
||||||
|
}
|
||||||
|
|
||||||
|
vn := demoteCandidate{}
|
||||||
|
err := msgp.Decode(&buf, &vn)
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
buf.Reset()
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
err = msgp.NewReader(&buf).Skip()
|
||||||
|
if err != nil {
|
||||||
|
t.Error(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkEncodedemoteCandidate(b *testing.B) {
|
||||||
|
v := demoteCandidate{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
en := msgp.NewWriter(msgp.Nowhere)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
v.EncodeMsg(en)
|
||||||
|
}
|
||||||
|
en.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
func BenchmarkDecodedemoteCandidate(b *testing.B) {
|
||||||
|
v := demoteCandidate{}
|
||||||
|
var buf bytes.Buffer
|
||||||
|
msgp.Encode(&buf, &v)
|
||||||
|
b.SetBytes(int64(buf.Len()))
|
||||||
|
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||||
|
dc := msgp.NewReader(rd)
|
||||||
|
b.ReportAllocs()
|
||||||
|
b.ResetTimer()
|
||||||
|
for i := 0; i < b.N; i++ {
|
||||||
|
err := v.DecodeMsg(dc)
|
||||||
|
if err != nil {
|
||||||
|
b.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
|
||||||
|
package cmd
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/config/ilm"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAccessEntryRollAndHits(t *testing.T) {
|
||||||
|
entry := accessEntry{Bins: []uint32{3, 2, 1}, HeadAt: 100, LastAt: 107}
|
||||||
|
entry.rollTo(120, 10, 3)
|
||||||
|
want := []uint32{0, 0, 3}
|
||||||
|
for i := range want {
|
||||||
|
if entry.Bins[i] != want[i] {
|
||||||
|
t.Fatalf("bins = %v, want %v", entry.Bins, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if entry.HeadAt != 120 || entry.LastAt != 107 {
|
||||||
|
t.Fatalf("head/last = %d/%d", entry.HeadAt, entry.LastAt)
|
||||||
|
}
|
||||||
|
|
||||||
|
entry = accessEntry{Bins: []uint32{10, 20, 30}, HeadAt: 120}
|
||||||
|
if got := entry.hits(20*time.Second, 10); got != 30 {
|
||||||
|
t.Fatalf("20s hits = %d, want 30", got)
|
||||||
|
}
|
||||||
|
if got := entry.hits(21*time.Second, 10); got != 60 {
|
||||||
|
t.Fatalf("21s hits = %d, want 60", got)
|
||||||
|
}
|
||||||
|
entry.rollTo(200, 10, 3)
|
||||||
|
if got := entry.total(); got != 0 {
|
||||||
|
t.Fatalf("expired total = %d, want 0", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessEntryMergeSaturates(t *testing.T) {
|
||||||
|
entry := accessEntry{Bins: []uint32{math.MaxUint32 - 1}}
|
||||||
|
entry.mergeFrom(accessEntry{Bins: []uint32{10}, LastAt: 50})
|
||||||
|
if entry.Bins[0] != math.MaxUint32 || entry.LastAt != 50 {
|
||||||
|
t.Fatalf("merged entry = %+v", entry)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTrackerEvictsColdest(t *testing.T) {
|
||||||
|
tracker := newAccessTracker()
|
||||||
|
live := map[string]accessEntry{
|
||||||
|
"a": {Bins: []uint32{1}, HeadAt: 100},
|
||||||
|
"b": {Bins: []uint32{5}, HeadAt: 100},
|
||||||
|
"c": {Bins: []uint32{3}, HeadAt: 100},
|
||||||
|
}
|
||||||
|
tracker.evict(live, 100, 10, ilm.Config{AccessBins: 1, AccessMaxTracked: 2})
|
||||||
|
if len(live) != 2 {
|
||||||
|
t.Fatalf("len = %d, want 2", len(live))
|
||||||
|
}
|
||||||
|
if _, ok := live["a"]; ok {
|
||||||
|
t.Fatal("coldest entry was retained")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessKeyRoundTrip(t *testing.T) {
|
||||||
|
key := accessKey("bucket", "a/b/c")
|
||||||
|
bucket, object, ok := splitAccessKey(key)
|
||||||
|
if !ok || bucket != "bucket" || object != "a/b/c" {
|
||||||
|
t.Fatalf("split %q = %q/%q/%v", key, bucket, object, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessShardFresh(t *testing.T) {
|
||||||
|
const now = int64(1000)
|
||||||
|
if !accessShardFresh(now, accessShard{UpdatedAt: 950}, 100) {
|
||||||
|
t.Fatal("fresh shard rejected")
|
||||||
|
}
|
||||||
|
if accessShardFresh(now, accessShard{UpdatedAt: 899}, 100) {
|
||||||
|
t.Fatal("stale shard accepted")
|
||||||
|
}
|
||||||
|
if accessShardFresh(now, accessShard{UpdatedAt: 1101}, 100) {
|
||||||
|
t.Fatal("far-future shard accepted")
|
||||||
|
}
|
||||||
|
if accessShardFresh(now, accessShard{}, 100) {
|
||||||
|
t.Fatal("zero timestamp accepted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTrackerRestoresDemoteCandidates(t *testing.T) {
|
||||||
|
tracker := newAccessTracker()
|
||||||
|
candidate := demoteCandidate{Bucket: "bucket", Object: "object", Pool: 1}
|
||||||
|
tracker.restoreDemoteCandidates([]demoteCandidate{candidate})
|
||||||
|
got := tracker.takeDemoteCandidates()
|
||||||
|
if len(got) != 1 || got[0] != candidate {
|
||||||
|
t.Fatalf("restored candidates = %+v, want %+v", got, candidate)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -19,6 +19,7 @@ package cmd
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"sync"
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
"github.com/minio/minio/internal/config/ilm"
|
"github.com/minio/minio/internal/config/ilm"
|
||||||
)
|
)
|
||||||
@@ -27,6 +28,16 @@ var globalILMConfig = ilmConfig{
|
|||||||
cfg: ilm.Config{
|
cfg: ilm.Config{
|
||||||
ExpirationWorkers: 100,
|
ExpirationWorkers: 100,
|
||||||
TransitionWorkers: 100,
|
TransitionWorkers: 100,
|
||||||
|
// Access tiering stays off until configured, but the counter
|
||||||
|
// geometry must be sane from the start: the tracker divides by
|
||||||
|
// AccessBinWidth before any config is loaded.
|
||||||
|
AccessPromoteWatermark: 85,
|
||||||
|
AccessBinWidth: time.Minute,
|
||||||
|
AccessBins: 12,
|
||||||
|
AccessFlush: time.Minute,
|
||||||
|
AccessMinResidency: 24 * time.Hour,
|
||||||
|
AccessWorkers: 10,
|
||||||
|
AccessMaxTracked: 1000000,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -49,6 +60,25 @@ func (c *ilmConfig) getTransitionWorkers() int {
|
|||||||
return c.cfg.TransitionWorkers
|
return c.cfg.TransitionWorkers
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// accessCfg returns a copy of the access tiering settings. Callers take the
|
||||||
|
// whole struct rather than one getter per field because the promotion and
|
||||||
|
// demotion paths need a consistent view of several knobs at once.
|
||||||
|
func (c *ilmConfig) accessCfg() ilm.Config {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
return c.cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
// accessTieringEnabled is the cheap gate used on the scanner path.
|
||||||
|
func (c *ilmConfig) accessTieringEnabled() bool {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
_, ok := c.cfg.HotPool()
|
||||||
|
return ok
|
||||||
|
}
|
||||||
|
|
||||||
func (c *ilmConfig) update(cfg ilm.Config) {
|
func (c *ilmConfig) update(cfg ilm.Config) {
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
defer c.mu.Unlock()
|
defer c.mu.Unlock()
|
||||||
|
|||||||
@@ -19,11 +19,12 @@ func _() {
|
|||||||
_ = x[lcEventSrc_s3PutObject-8]
|
_ = x[lcEventSrc_s3PutObject-8]
|
||||||
_ = x[lcEventSrc_s3CopyObject-9]
|
_ = x[lcEventSrc_s3CopyObject-9]
|
||||||
_ = x[lcEventSrc_s3CompleteMultipartUpload-10]
|
_ = x[lcEventSrc_s3CompleteMultipartUpload-10]
|
||||||
|
_ = x[lcEventSrc_AccessTier-11]
|
||||||
}
|
}
|
||||||
|
|
||||||
const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUpload"
|
const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUploadAccessTier"
|
||||||
|
|
||||||
var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109}
|
var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109, 119}
|
||||||
|
|
||||||
func (i lcEventSrc) String() string {
|
func (i lcEventSrc) String() string {
|
||||||
idx := int(i) - 0
|
idx := int(i) - 0
|
||||||
|
|||||||
@@ -26,6 +26,17 @@ const (
|
|||||||
transitionActiveTasks = "transition_active_tasks"
|
transitionActiveTasks = "transition_active_tasks"
|
||||||
transitionPendingTasks = "transition_pending_tasks"
|
transitionPendingTasks = "transition_pending_tasks"
|
||||||
transitionMissedImmediateTasks = "transition_missed_immediate_tasks"
|
transitionMissedImmediateTasks = "transition_missed_immediate_tasks"
|
||||||
|
accessTierActiveTasks = "access_tier_active_tasks"
|
||||||
|
accessTierPendingTasks = "access_tier_pending_tasks"
|
||||||
|
accessTierPromotionsTotal = "access_tier_promotions_total"
|
||||||
|
accessTierDemotionsTotal = "access_tier_demotions_total"
|
||||||
|
accessTierBytesMovedTotal = "access_tier_bytes_moved_total"
|
||||||
|
accessTierFailuresTotal = "access_tier_failures_total"
|
||||||
|
accessTierSkippedWatermark = "access_tier_skipped_watermark_total"
|
||||||
|
accessTierSkippedMaxSize = "access_tier_skipped_max_size_total"
|
||||||
|
accessTierSkippedQuota = "access_tier_skipped_bucket_quota_total"
|
||||||
|
accessTierHotBytes = "access_tier_hot_bytes"
|
||||||
|
accessTierSamplesDropped = "access_tier_samples_dropped_total"
|
||||||
versionsScanned = "versions_scanned"
|
versionsScanned = "versions_scanned"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,6 +45,17 @@ var (
|
|||||||
ilmTransitionActiveTasksMD = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks")
|
ilmTransitionActiveTasksMD = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks")
|
||||||
ilmTransitionPendingTasksMD = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue")
|
ilmTransitionPendingTasksMD = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue")
|
||||||
ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks")
|
ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks")
|
||||||
|
ilmAccessTierActiveTasksMD = NewGaugeMD(accessTierActiveTasks, "Number of active access-tier pool moves")
|
||||||
|
ilmAccessTierPendingTasksMD = NewGaugeMD(accessTierPendingTasks, "Number of pending access-tier pool moves")
|
||||||
|
ilmAccessTierPromotionsTotalMD = NewCounterMD(accessTierPromotionsTotal, "Total objects promoted by access-tier ILM")
|
||||||
|
ilmAccessTierDemotionsTotalMD = NewCounterMD(accessTierDemotionsTotal, "Total objects demoted by access-tier ILM")
|
||||||
|
ilmAccessTierBytesMovedTotalMD = NewCounterMD(accessTierBytesMovedTotal, "Total logical bytes moved by access-tier ILM")
|
||||||
|
ilmAccessTierFailuresTotalMD = NewCounterMD(accessTierFailuresTotal, "Total failed access-tier ILM moves")
|
||||||
|
ilmAccessTierSkippedWatermarkMD = NewCounterMD(accessTierSkippedWatermark, "Promotions skipped because the hot pool reached its watermark")
|
||||||
|
ilmAccessTierSkippedMaxSizeMD = NewCounterMD(accessTierSkippedMaxSize, "Promotions skipped because the cluster hot-tier size cap was reached")
|
||||||
|
ilmAccessTierSkippedQuotaMD = NewCounterMD(accessTierSkippedQuota, "Promotions skipped because the bucket hot-tier quota was reached")
|
||||||
|
ilmAccessTierHotBytesMD = NewGaugeMD(accessTierHotBytes, "Logical bytes currently accounted to the hot tier", "bucket")
|
||||||
|
ilmAccessTierSamplesDroppedMD = NewCounterMD(accessTierSamplesDropped, "GET samples dropped because the access tracker queue was full")
|
||||||
ilmVersionsScannedMD = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start")
|
ilmVersionsScannedMD = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start")
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -47,6 +69,21 @@ func loadILMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error {
|
|||||||
m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks()))
|
m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks()))
|
||||||
m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks()))
|
m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks()))
|
||||||
}
|
}
|
||||||
|
if globalAccessTierState != nil {
|
||||||
|
m.Set(accessTierActiveTasks, float64(globalAccessTierState.ActiveTasks()))
|
||||||
|
m.Set(accessTierPendingTasks, float64(globalAccessTierState.PendingTasks()))
|
||||||
|
m.Set(accessTierPromotionsTotal, float64(globalAccessTierState.promotions.Load()))
|
||||||
|
m.Set(accessTierDemotionsTotal, float64(globalAccessTierState.demotions.Load()))
|
||||||
|
m.Set(accessTierBytesMovedTotal, float64(globalAccessTierState.bytesMoved.Load()))
|
||||||
|
m.Set(accessTierFailuresTotal, float64(globalAccessTierState.failures.Load()))
|
||||||
|
m.Set(accessTierSkippedWatermark, float64(globalAccessTierState.skippedWatermark.Load()))
|
||||||
|
m.Set(accessTierSkippedMaxSize, float64(globalAccessTierState.skippedMaxSize.Load()))
|
||||||
|
m.Set(accessTierSkippedQuota, float64(globalAccessTierState.skippedQuota.Load()))
|
||||||
|
for bucket, bytes := range globalAccessTierState.hotUsageSnapshot() {
|
||||||
|
m.Set(accessTierHotBytes, float64(bytes), "bucket", bucket)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
m.Set(accessTierSamplesDropped, float64(globalAccessTracker.dropped.Load()))
|
||||||
m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM)))
|
m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM)))
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -390,6 +390,17 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
|
|||||||
ilmTransitionActiveTasksMD,
|
ilmTransitionActiveTasksMD,
|
||||||
ilmTransitionPendingTasksMD,
|
ilmTransitionPendingTasksMD,
|
||||||
ilmTransitionMissedImmediateTasksMD,
|
ilmTransitionMissedImmediateTasksMD,
|
||||||
|
ilmAccessTierActiveTasksMD,
|
||||||
|
ilmAccessTierPendingTasksMD,
|
||||||
|
ilmAccessTierPromotionsTotalMD,
|
||||||
|
ilmAccessTierDemotionsTotalMD,
|
||||||
|
ilmAccessTierBytesMovedTotalMD,
|
||||||
|
ilmAccessTierFailuresTotalMD,
|
||||||
|
ilmAccessTierSkippedWatermarkMD,
|
||||||
|
ilmAccessTierSkippedMaxSizeMD,
|
||||||
|
ilmAccessTierSkippedQuotaMD,
|
||||||
|
ilmAccessTierHotBytesMD,
|
||||||
|
ilmAccessTierSamplesDroppedMD,
|
||||||
ilmVersionsScannedMD,
|
ilmVersionsScannedMD,
|
||||||
},
|
},
|
||||||
loadILMMetrics,
|
loadILMMetrics,
|
||||||
|
|||||||
@@ -117,6 +117,10 @@ type ObjectOptions struct {
|
|||||||
SkipRebalancing bool
|
SkipRebalancing bool
|
||||||
|
|
||||||
SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same
|
SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same
|
||||||
|
// DstPoolIdx forces a data-movement write onto a specific server pool.
|
||||||
|
// It is ignored unless DataMovement is true; a pointer keeps pool zero
|
||||||
|
// distinguishable from the unset value.
|
||||||
|
DstPoolIdx *int
|
||||||
|
|
||||||
DataMovement bool // indicates an going decommisionning or rebalacing
|
DataMovement bool // indicates an going decommisionning or rebalacing
|
||||||
|
|
||||||
|
|||||||
@@ -563,6 +563,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
globalAccessTracker.note(bucket, object)
|
||||||
|
|
||||||
// Notify object accessed via a GET request.
|
// Notify object accessed via a GET request.
|
||||||
sendEvent(eventArgs{
|
sendEvent(eventArgs{
|
||||||
EventName: event.ObjectAccessedGet,
|
EventName: event.ObjectAccessedGet,
|
||||||
|
|||||||
@@ -500,6 +500,7 @@ func initAllSubsystems(ctx context.Context) {
|
|||||||
globalTierConfigMgr = NewTierConfigMgr()
|
globalTierConfigMgr = NewTierConfigMgr()
|
||||||
|
|
||||||
globalTransitionState = newTransitionState(GlobalContext)
|
globalTransitionState = newTransitionState(GlobalContext)
|
||||||
|
globalAccessTierState = newAccessTierState(GlobalContext)
|
||||||
globalSiteResyncMetrics = newSiteResyncMetrics(GlobalContext)
|
globalSiteResyncMetrics = newSiteResyncMetrics(GlobalContext)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1057,6 +1058,10 @@ func serverMain(ctx *cli.Context) {
|
|||||||
bootstrapTrace("globalTransitionState.Init", func() {
|
bootstrapTrace("globalTransitionState.Init", func() {
|
||||||
globalTransitionState.Init(newObject)
|
globalTransitionState.Init(newObject)
|
||||||
})
|
})
|
||||||
|
bootstrapTrace("globalAccessTierState.Init", func() {
|
||||||
|
globalAccessTierState.Init(newObject)
|
||||||
|
go globalAccessTracker.run(GlobalContext, newObject)
|
||||||
|
})
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
// Initialize transition tier configuration manager
|
// Initialize transition tier configuration manager
|
||||||
|
|||||||
@@ -583,6 +583,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
|
|||||||
}
|
}
|
||||||
|
|
||||||
poolIdx, setIdx, _ := s.GetDiskLoc()
|
poolIdx, setIdx, _ := s.GetDiskLoc()
|
||||||
|
hotPool, hotPoolOK := globalILMConfig.accessCfg().HotPool()
|
||||||
|
|
||||||
disks, err := objAPI.GetDisks(poolIdx, setIdx)
|
disks, err := objAPI.GetDisks(poolIdx, setIdx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -592,6 +593,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
|
|||||||
cache.Info.updates = updates
|
cache.Info.updates = updates
|
||||||
|
|
||||||
dataUsageInfo, err := scanDataFolder(ctx, disks, s, cache, func(item scannerItem) (sizeSummary, error) {
|
dataUsageInfo, err := scanDataFolder(ctx, disks, s, cache, func(item scannerItem) (sizeSummary, error) {
|
||||||
|
item.poolIdx = poolIdx
|
||||||
// Look for `xl.meta/xl.json' at the leaf.
|
// Look for `xl.meta/xl.json' at the leaf.
|
||||||
if !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFile) &&
|
if !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFile) &&
|
||||||
!strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFileV1) {
|
!strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFileV1) {
|
||||||
@@ -655,6 +657,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
|
|||||||
sizeS.versions++
|
sizeS.versions++
|
||||||
}
|
}
|
||||||
sizeS.totalSize += sz
|
sizeS.totalSize += sz
|
||||||
|
if hotPoolOK && poolIdx == hotPool {
|
||||||
|
sizeS.hotTierSize += sz
|
||||||
|
}
|
||||||
|
|
||||||
// Skip tier accounting if object version is a delete-marker or a free-version
|
// Skip tier accounting if object version is a delete-marker or a free-version
|
||||||
// tracking deleted transitioned objects
|
// tracking deleted transitioned objects
|
||||||
|
|||||||
@@ -226,6 +226,128 @@ aws s3api restore-object --bucket srcbucket \
|
|||||||
|
|
||||||
Note that transition event notification is a Silo extension.
|
Note that transition event notification is a Silo extension.
|
||||||
|
|
||||||
|
## 5. Access-based tiering between server pools
|
||||||
|
|
||||||
|
Silo can move frequently read objects to a faster server pool and return them
|
||||||
|
to a slower pool after they become idle. This is different from remote ILM
|
||||||
|
transition: the object remains a native local object and all versions move
|
||||||
|
together.
|
||||||
|
|
||||||
|
Access tiering requires at least two server pools. Pool indices follow the
|
||||||
|
order on the server command line:
|
||||||
|
|
||||||
|
~~~sh
|
||||||
|
silo server /srv/nvme{1...4} /srv/hdd{1...8}
|
||||||
|
# pool 0 (fast) pool 1 (slow)
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Server pools are erasure-coding expansion units, not individual drives. Each
|
||||||
|
pool should consist of internally homogeneous media.
|
||||||
|
|
||||||
|
The feature is disabled by default. Configure the topology and safety limits
|
||||||
|
with `mc admin config set`; ILM is a dynamic subsystem, so this applies without
|
||||||
|
a restart. Environment variables (`MINIO_ILM_ACCESS_TIERING`,
|
||||||
|
`MINIO_ILM_ACCESS_POOLS`, `MINIO_ILM_ACCESS_MAX_SIZE`,
|
||||||
|
`MINIO_ILM_ACCESS_PROMOTE_WATERMARK`, `MINIO_ILM_ACCESS_BIN_WIDTH`,
|
||||||
|
`MINIO_ILM_ACCESS_BINS`, `MINIO_ILM_ACCESS_FLUSH`,
|
||||||
|
`MINIO_ILM_ACCESS_MIN_RESIDENCY`, `MINIO_ILM_ACCESS_WORKERS`,
|
||||||
|
`MINIO_ILM_ACCESS_MAX_TRACKED`) are read at process start and override the
|
||||||
|
stored config.
|
||||||
|
|
||||||
|
~~~sh
|
||||||
|
mc admin config set local ilm \
|
||||||
|
access_tiering=on \
|
||||||
|
access_pools="0,1" \
|
||||||
|
access_max_size="2TiB" \
|
||||||
|
access_promote_watermark=85 \
|
||||||
|
access_bin_width=1m \
|
||||||
|
access_bins=12 \
|
||||||
|
access_flush=1m \
|
||||||
|
access_min_residency=24h \
|
||||||
|
access_workers=10 \
|
||||||
|
access_max_tracked=1000000
|
||||||
|
~~~
|
||||||
|
|
||||||
|
The pool list is ordered hottest to coldest. With three or more pools,
|
||||||
|
promotion always targets the first index and demotion always targets the last;
|
||||||
|
intermediate pools are not hop targets. An access_max_size value of zero means
|
||||||
|
no cluster-wide logical-byte cap. Promotion also stops when the hottest pool
|
||||||
|
reaches access_promote_watermark.
|
||||||
|
|
||||||
|
New PUTs still land via the usual free-space pool picker; they are not steered
|
||||||
|
onto the cold pool. Size the capacity pool larger than the hot pool so new
|
||||||
|
objects tend to land there.
|
||||||
|
|
||||||
|
Add an AccessTransition to the bucket lifecycle XML:
|
||||||
|
|
||||||
|
~~~xml
|
||||||
|
<LifecycleConfiguration>
|
||||||
|
<AccessTierQuota>500GiB</AccessTierQuota>
|
||||||
|
<Rule>
|
||||||
|
<ID>hot-logs</ID>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter>
|
||||||
|
<And>
|
||||||
|
<Prefix>logs/</Prefix>
|
||||||
|
<ObjectSizeGreaterThan>65536</ObjectSizeGreaterThan>
|
||||||
|
</And>
|
||||||
|
</Filter>
|
||||||
|
<AccessTransition>
|
||||||
|
<Window>10m</Window>
|
||||||
|
<PromoteAfterAccesses>100</PromoteAfterAccesses>
|
||||||
|
<DemoteAfterAccesses>5</DemoteAfterAccesses>
|
||||||
|
<DemoteAfterIdle>24h</DemoteAfterIdle>
|
||||||
|
</AccessTransition>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>
|
||||||
|
~~~
|
||||||
|
|
||||||
|
This promotes a matching object after 100 successful GETs in 10 minutes. An
|
||||||
|
object becomes eligible to return to the coldest configured pool only after
|
||||||
|
access tiering has already moved it (the `x-minio-internal-ilm-atier` stamp),
|
||||||
|
it has stayed put for the server-wide minimum residency, it has been idle at
|
||||||
|
least 24 hours, and it has no more than 5 GETs in the window. Objects that
|
||||||
|
landed on the hot pool via a normal PUT never demote. Prefix, tag, and
|
||||||
|
object-size lifecycle filters are honored.
|
||||||
|
|
||||||
|
Access-based moves are a parallel path: they are not lifecycle `Eval` actions
|
||||||
|
and do not appear in S3 prediction headers. If the same object is also due
|
||||||
|
for age-based remote `Transition` or expiry, that scanner action wins and
|
||||||
|
demotion discovery is skipped for that pass; promotions still run from the
|
||||||
|
GET tracker. Site replication copies expiry rules only, same as remote
|
||||||
|
Transition, so AccessTransition stays local to the cluster.
|
||||||
|
|
||||||
|
AccessTierQuota is an optional bucket-wide cap. Promotion checks, in order:
|
||||||
|
|
||||||
|
1. hot-pool used percentage;
|
||||||
|
2. cluster-wide access_max_size;
|
||||||
|
3. bucket AccessTierQuota.
|
||||||
|
|
||||||
|
Demotion is not blocked by these caps and is processed before promotion.
|
||||||
|
Access moves pause during rebalance or decommission, never target a suspended
|
||||||
|
pool, skip remotely transitioned objects and objects with excessive version
|
||||||
|
counts, and recheck eligibility while holding the object namespace lock.
|
||||||
|
|
||||||
|
The hit counter is intentionally best effort. Only successfully served GET
|
||||||
|
requests count; HEAD requests do not. Counters are merged across nodes and
|
||||||
|
bounded by access_max_tracked. A rule window longer than
|
||||||
|
access_bin_width multiplied by access_bins is clamped to retained history.
|
||||||
|
|
||||||
|
AccessTransition and AccessTierQuota are Silo lifecycle extensions. A stock
|
||||||
|
AWS SDK that reads and rewrites the lifecycle configuration may discard
|
||||||
|
unknown fields. Use a raw signed S3 PUT lifecycle request, such as
|
||||||
|
[setup_ilm_access_tiering.sh](setup_ilm_access_tiering.sh), when installing
|
||||||
|
the rule. Save the XML above as `rule.xml`, then run:
|
||||||
|
|
||||||
|
~~~sh
|
||||||
|
AWS_ACCESS_KEY_ID=minioadmin AWS_SECRET_ACCESS_KEY=minioadmin \
|
||||||
|
./setup_ilm_access_tiering.sh http://127.0.0.1:9000 testbucket us-east-1 rule.xml
|
||||||
|
~~~
|
||||||
|
|
||||||
|
Access-tier activity is exposed under /minio/metrics/v3/ilm, including move
|
||||||
|
counts, moved bytes, queue depth, hot bytes per bucket, failed moves, dropped
|
||||||
|
GET samples, and separate skip counters for each capacity limit.
|
||||||
|
|
||||||
## Explore Further
|
## Explore Further
|
||||||
|
|
||||||
- [MinIO Go client API reference (S3-compatible SDK)](https://pkg.go.dev/github.com/minio/minio-go/v7)
|
- [MinIO Go client API reference (S3-compatible SDK)](https://pkg.go.dev/github.com/minio/minio-go/v7)
|
||||||
|
|||||||
+37
@@ -0,0 +1,37 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# Install a lifecycle XML document without an SDK normalizing away Silo's
|
||||||
|
# AccessTransition and AccessTierQuota extension elements.
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
if [ "$#" -ne 4 ]; then
|
||||||
|
echo "usage: AWS_ACCESS_KEY_ID=... AWS_SECRET_ACCESS_KEY=... $0 ENDPOINT BUCKET REGION LIFECYCLE_XML" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
endpoint=$1
|
||||||
|
bucket=$2
|
||||||
|
region=$3
|
||||||
|
lifecycle_file=$4
|
||||||
|
|
||||||
|
: "$AWS_ACCESS_KEY_ID"
|
||||||
|
: "$AWS_SECRET_ACCESS_KEY"
|
||||||
|
|
||||||
|
if [ ! -r "$lifecycle_file" ]; then
|
||||||
|
echo "cannot read lifecycle document: $lifecycle_file" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
content_md5=$(openssl dgst -md5 -binary "$lifecycle_file" | openssl base64)
|
||||||
|
endpoint=$(printf '%s' "$endpoint" | sed 's:/*$::')
|
||||||
|
|
||||||
|
curl --fail-with-body --silent --show-error \
|
||||||
|
--request PUT \
|
||||||
|
--aws-sigv4 "aws:amz:$region:s3" \
|
||||||
|
--user "$AWS_ACCESS_KEY_ID:$AWS_SECRET_ACCESS_KEY" \
|
||||||
|
--header "Content-MD5: $content_md5" \
|
||||||
|
--header "Content-Type: application/xml" \
|
||||||
|
--data-binary "@$lifecycle_file" \
|
||||||
|
"$endpoint/$bucket?lifecycle"
|
||||||
|
|
||||||
|
echo "installed lifecycle configuration on $bucket"
|
||||||
@@ -225,6 +225,17 @@ For deployments with [bucket](https://silo.pgsty.com/administration/bucket-repli
|
|||||||
| `minio_node_ilm_transition_active_tasks` | Number of active ILM transition tasks. |
|
| `minio_node_ilm_transition_active_tasks` | Number of active ILM transition tasks. |
|
||||||
| `minio_node_ilm_transition_pending_tasks` | Number of pending ILM transition tasks in the queue. |
|
| `minio_node_ilm_transition_pending_tasks` | Number of pending ILM transition tasks in the queue. |
|
||||||
| `minio_node_ilm_transition_missed_immediate_tasks` | Number of missed immediate ILM transition tasks. |
|
| `minio_node_ilm_transition_missed_immediate_tasks` | Number of missed immediate ILM transition tasks. |
|
||||||
|
| `minio_node_ilm_access_tier_active_tasks` | Number of active access-tier pool moves. |
|
||||||
|
| `minio_node_ilm_access_tier_pending_tasks` | Number of pending access-tier pool moves. |
|
||||||
|
| `minio_node_ilm_access_tier_promotions_total` | Total objects promoted by access-tier ILM. |
|
||||||
|
| `minio_node_ilm_access_tier_demotions_total` | Total objects demoted by access-tier ILM. |
|
||||||
|
| `minio_node_ilm_access_tier_bytes_moved_total` | Total logical bytes moved by access-tier ILM. |
|
||||||
|
| `minio_node_ilm_access_tier_failures_total` | Total failed access-tier ILM moves. |
|
||||||
|
| `minio_node_ilm_access_tier_skipped_watermark_total` | Promotions skipped because the hot pool reached its watermark. |
|
||||||
|
| `minio_node_ilm_access_tier_skipped_max_size_total` | Promotions skipped because the cluster hot-tier size cap was reached. |
|
||||||
|
| `minio_node_ilm_access_tier_skipped_bucket_quota_total` | Promotions skipped because the bucket hot-tier quota was reached. |
|
||||||
|
| `minio_node_ilm_access_tier_hot_bytes` | Logical bytes currently accounted to the hot tier, labeled by bucket. |
|
||||||
|
| `minio_node_ilm_access_tier_samples_dropped_total` | GET samples dropped because the access tracker queue was full. |
|
||||||
| `minio_node_ilm_versions_scanned` | Total number of object versions checked for ilm actions since server start. |
|
| `minio_node_ilm_versions_scanned` | Total number of object versions checked for ilm actions since server start. |
|
||||||
| `minio_node_ilm_action_count_delete_action` | Total action outcome of lifecycle checks since server start for deleting object |
|
| `minio_node_ilm_action_count_delete_action` | Total action outcome of lifecycle checks since server start for deleting object |
|
||||||
| `minio_node_ilm_action_count_delete_version_action` | Total action outcome of lifecycle checks since server start for deleting a version |
|
| `minio_node_ilm_action_count_delete_version_action` | Total action outcome of lifecycle checks since server start for deleting a version |
|
||||||
|
|||||||
@@ -0,0 +1,129 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
errAccessInvalidDuration = Errorf("Window and DemoteAfterIdle must be valid Go durations, e.g. 10m or 24h")
|
||||||
|
errAccessInvalidWindow = Errorf("Window must be a positive duration with AccessTransition")
|
||||||
|
errAccessInvalidPromote = Errorf("PromoteAfterAccesses must be a positive integer with AccessTransition")
|
||||||
|
errAccessInvalidDemote = Errorf("DemoteAfterAccesses must be smaller than PromoteAfterAccesses and 0 or greater")
|
||||||
|
errAccessInvalidIdle = Errorf("DemoteAfterIdle must be a positive duration no shorter than Window")
|
||||||
|
errAccessInvalidQuotaSize = Errorf("AccessTierQuota must be a valid size, e.g. 500GiB")
|
||||||
|
)
|
||||||
|
|
||||||
|
// Duration is a time.Duration that marshals to and from an XML element
|
||||||
|
// holding a Go duration string, e.g. <Window>10m</Window>.
|
||||||
|
type Duration time.Duration
|
||||||
|
|
||||||
|
// UnmarshalXML parses a duration string such as "10m" or "24h".
|
||||||
|
func (d *Duration) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error {
|
||||||
|
var s string
|
||||||
|
if err := dec.DecodeElement(&s, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
dur, err := time.ParseDuration(s)
|
||||||
|
if err != nil {
|
||||||
|
return errAccessInvalidDuration
|
||||||
|
}
|
||||||
|
*d = Duration(dur)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalXML encodes a non-zero duration, and nothing otherwise.
|
||||||
|
func (d Duration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
|
||||||
|
if d == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return enc.EncodeElement(time.Duration(d).String(), start)
|
||||||
|
}
|
||||||
|
|
||||||
|
// D returns the value as a time.Duration.
|
||||||
|
func (d Duration) D() time.Duration {
|
||||||
|
return time.Duration(d)
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessTransition is a Silo extension to the S3 lifecycle rule. It relocates
|
||||||
|
// an object between server pools based on how often it is read, rather than on
|
||||||
|
// its age: an object read at least PromoteAfterAccesses times within Window
|
||||||
|
// moves to the fastest configured pool, and moves back once it has been idle
|
||||||
|
// for DemoteAfterIdle and its windowed hit count has fallen to
|
||||||
|
// DemoteAfterAccesses or below.
|
||||||
|
//
|
||||||
|
// The gap between the two thresholds, together with DemoteAfterIdle and the
|
||||||
|
// server-side access_min_residency, is what keeps an object from oscillating
|
||||||
|
// between pools.
|
||||||
|
type AccessTransition struct {
|
||||||
|
XMLName xml.Name `xml:"AccessTransition"`
|
||||||
|
Window Duration `xml:"Window,omitempty"`
|
||||||
|
PromoteAfterAccesses int `xml:"PromoteAfterAccesses,omitempty"`
|
||||||
|
DemoteAfterAccesses int `xml:"DemoteAfterAccesses,omitempty"`
|
||||||
|
DemoteAfterIdle Duration `xml:"DemoteAfterIdle,omitempty"`
|
||||||
|
|
||||||
|
set bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// IsNull returns true if no usable access transition is configured.
|
||||||
|
func (a AccessTransition) IsNull() bool {
|
||||||
|
return !a.set || a.PromoteAfterAccesses <= 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// MarshalXML encodes an AccessTransition element, and nothing if unset.
|
||||||
|
func (a AccessTransition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
|
||||||
|
if !a.set {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
type accessTransitionWrapper AccessTransition
|
||||||
|
return enc.EncodeElement(accessTransitionWrapper(a), start)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalXML decodes an AccessTransition element.
|
||||||
|
func (a *AccessTransition) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error {
|
||||||
|
type accessTransitionWrapper AccessTransition
|
||||||
|
var atw accessTransitionWrapper
|
||||||
|
if err := dec.DecodeElement(&atw, &start); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
*a = AccessTransition(atw)
|
||||||
|
a.set = true
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Validate checks the AccessTransition element.
|
||||||
|
func (a AccessTransition) Validate() error {
|
||||||
|
if !a.set {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
if a.Window <= 0 {
|
||||||
|
return errAccessInvalidWindow
|
||||||
|
}
|
||||||
|
if a.PromoteAfterAccesses <= 0 {
|
||||||
|
return errAccessInvalidPromote
|
||||||
|
}
|
||||||
|
if a.DemoteAfterAccesses < 0 || a.DemoteAfterAccesses >= a.PromoteAfterAccesses {
|
||||||
|
return errAccessInvalidDemote
|
||||||
|
}
|
||||||
|
if a.DemoteAfterIdle <= 0 || a.DemoteAfterIdle < a.Window {
|
||||||
|
return errAccessInvalidIdle
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,219 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
//
|
||||||
|
// This file is part of MinIO Object Storage stack
|
||||||
|
//
|
||||||
|
// This program is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// This program is distributed in the hope that it will be useful
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Affero General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Affero General Public License
|
||||||
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package lifecycle
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"encoding/xml"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/bucket/object/lock"
|
||||||
|
)
|
||||||
|
|
||||||
|
const accessTieringXML = `<LifecycleConfiguration>
|
||||||
|
<AccessTierQuota>500GiB</AccessTierQuota>
|
||||||
|
<Rule>
|
||||||
|
<ID>hot-logs</ID>
|
||||||
|
<Status>Enabled</Status>
|
||||||
|
<Filter><And><Prefix>logs/</Prefix><ObjectSizeGreaterThan>65536</ObjectSizeGreaterThan></And></Filter>
|
||||||
|
<AccessTransition>
|
||||||
|
<Window>10m</Window>
|
||||||
|
<PromoteAfterAccesses>100</PromoteAfterAccesses>
|
||||||
|
<DemoteAfterAccesses>5</DemoteAfterAccesses>
|
||||||
|
<DemoteAfterIdle>24h</DemoteAfterIdle>
|
||||||
|
</AccessTransition>
|
||||||
|
</Rule>
|
||||||
|
</LifecycleConfiguration>`
|
||||||
|
|
||||||
|
func TestAccessTransitionParse(t *testing.T) {
|
||||||
|
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse: %v", err)
|
||||||
|
}
|
||||||
|
if err := lc.Validate(lock.Retention{}); err != nil {
|
||||||
|
t.Fatalf("validate: %v", err)
|
||||||
|
}
|
||||||
|
if got := lc.AccessQuotaBytes(); got != 500*1024*1024*1024 {
|
||||||
|
t.Fatalf("quota = %d, want %d", got, 500*1024*1024*1024)
|
||||||
|
}
|
||||||
|
if !lc.HasAccessTransition() {
|
||||||
|
t.Fatal("HasAccessTransition = false, want true")
|
||||||
|
}
|
||||||
|
at := lc.Rules[0].AccessTransition
|
||||||
|
if at.Window.D() != 10*time.Minute {
|
||||||
|
t.Fatalf("window = %v, want 10m", at.Window.D())
|
||||||
|
}
|
||||||
|
if at.DemoteAfterIdle.D() != 24*time.Hour {
|
||||||
|
t.Fatalf("idle = %v, want 24h", at.DemoteAfterIdle.D())
|
||||||
|
}
|
||||||
|
if at.PromoteAfterAccesses != 100 || at.DemoteAfterAccesses != 5 {
|
||||||
|
t.Fatalf("thresholds = %d/%d, want 100/5", at.PromoteAfterAccesses, at.DemoteAfterAccesses)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A round trip through Marshal must preserve both the rule element and the
|
||||||
|
// bucket-wide quota, since PutBucketLifecycle stores whatever we re-encode.
|
||||||
|
func TestAccessTransitionRoundTrip(t *testing.T) {
|
||||||
|
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse: %v", err)
|
||||||
|
}
|
||||||
|
buf, err := xml.Marshal(lc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
if !bytes.Contains(buf, []byte("<AccessTierQuota>500GiB</AccessTierQuota>")) {
|
||||||
|
t.Fatalf("quota lost in round trip: %s", buf)
|
||||||
|
}
|
||||||
|
got, err := ParseLifecycleConfig(bytes.NewReader(buf))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("reparse: %v", err)
|
||||||
|
}
|
||||||
|
if got.AccessQuotaBytes() != lc.AccessQuotaBytes() {
|
||||||
|
t.Fatalf("quota %d != %d", got.AccessQuotaBytes(), lc.AccessQuotaBytes())
|
||||||
|
}
|
||||||
|
if got.Rules[0].AccessTransition != lc.Rules[0].AccessTransition {
|
||||||
|
t.Fatalf("rule %+v != %+v", got.Rules[0].AccessTransition, lc.Rules[0].AccessTransition)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// A rule with no AccessTransition must not emit an empty element - otherwise
|
||||||
|
// every existing lifecycle config would change shape on rewrite.
|
||||||
|
func TestAccessTransitionUnsetNotMarshalled(t *testing.T) {
|
||||||
|
lc, err := ParseLifecycleConfig(strings.NewReader(`<LifecycleConfiguration><Rule>
|
||||||
|
<ID>old</ID><Status>Enabled</Status><Filter><Prefix>a/</Prefix></Filter>
|
||||||
|
<Expiration><Days>3</Days></Expiration></Rule></LifecycleConfiguration>`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse: %v", err)
|
||||||
|
}
|
||||||
|
buf, err := xml.Marshal(lc)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal: %v", err)
|
||||||
|
}
|
||||||
|
if bytes.Contains(buf, []byte("AccessTransition")) || bytes.Contains(buf, []byte("AccessTierQuota")) {
|
||||||
|
t.Fatalf("unset elements emitted: %s", buf)
|
||||||
|
}
|
||||||
|
if lc.HasAccessTransition() {
|
||||||
|
t.Fatal("HasAccessTransition = true for a plain expiry rule")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTransitionValidate(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
at AccessTransition
|
||||||
|
err error
|
||||||
|
}{
|
||||||
|
{"ok", AccessTransition{Window: Duration(10 * time.Minute), PromoteAfterAccesses: 100, DemoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Hour), set: true}, nil},
|
||||||
|
{"unset", AccessTransition{}, nil},
|
||||||
|
{"zero window", AccessTransition{PromoteAfterAccesses: 100, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidWindow},
|
||||||
|
{"zero promote", AccessTransition{Window: Duration(time.Minute), DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidPromote},
|
||||||
|
{"demote >= promote", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, DemoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidDemote},
|
||||||
|
{"negative demote", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, DemoteAfterAccesses: -1, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidDemote},
|
||||||
|
{"idle shorter than window", AccessTransition{Window: Duration(time.Hour), PromoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Minute), set: true}, errAccessInvalidIdle},
|
||||||
|
{"zero idle", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, set: true}, errAccessInvalidIdle},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if err := tc.at.Validate(); err != tc.err {
|
||||||
|
t.Fatalf("err = %v, want %v", err, tc.err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTierQuotaInvalid(t *testing.T) {
|
||||||
|
lc, err := ParseLifecycleConfig(strings.NewReader(`<LifecycleConfiguration>
|
||||||
|
<AccessTierQuota>not-a-size</AccessTierQuota>
|
||||||
|
<Rule><ID>r</ID><Status>Enabled</Status><Expiration><Days>3</Days></Expiration></Rule>
|
||||||
|
</LifecycleConfiguration>`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse: %v", err)
|
||||||
|
}
|
||||||
|
if err := lc.Validate(lock.Retention{}); err != errAccessInvalidQuotaSize {
|
||||||
|
t.Fatalf("err = %v, want %v", err, errAccessInvalidQuotaSize)
|
||||||
|
}
|
||||||
|
// An invalid quota that somehow reached the evaluator means "unlimited",
|
||||||
|
// never "zero bytes allowed".
|
||||||
|
if got := lc.AccessQuotaBytes(); got != 0 {
|
||||||
|
t.Fatalf("quota = %d, want 0 (unlimited)", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTransitionBadDuration(t *testing.T) {
|
||||||
|
_, err := ParseLifecycleConfig(strings.NewReader(`<LifecycleConfiguration><Rule>
|
||||||
|
<ID>r</ID><Status>Enabled</Status>
|
||||||
|
<AccessTransition><Window>ten minutes</Window><PromoteAfterAccesses>1</PromoteAfterAccesses></AccessTransition>
|
||||||
|
</Rule></LifecycleConfiguration>`))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected a parse error for a malformed duration")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessRule must reuse the standard rule filtering: prefix, tags, size and
|
||||||
|
// Status all have to be honored.
|
||||||
|
func TestAccessRuleFiltering(t *testing.T) {
|
||||||
|
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse: %v", err)
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
obj ObjectOpts
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{"match", ObjectOpts{Name: "logs/a.log", Size: 1 << 20, IsLatest: true}, true},
|
||||||
|
{"wrong prefix", ObjectOpts{Name: "data/a.log", Size: 1 << 20, IsLatest: true}, false},
|
||||||
|
{"too small", ObjectOpts{Name: "logs/a.log", Size: 1024, IsLatest: true}, false},
|
||||||
|
{"no name", ObjectOpts{Size: 1 << 20}, false},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, id, ok := lc.AccessRule(tc.obj)
|
||||||
|
if ok != tc.want {
|
||||||
|
t.Fatalf("ok = %v, want %v", ok, tc.want)
|
||||||
|
}
|
||||||
|
if ok && id != "hot-logs" {
|
||||||
|
t.Fatalf("ruleID = %q, want hot-logs", id)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
lc.Rules[0].Status = Disabled
|
||||||
|
if _, _, ok := lc.AccessRule(ObjectOpts{Name: "logs/a.log", Size: 1 << 20, IsLatest: true}); ok {
|
||||||
|
t.Fatal("disabled rule still matched")
|
||||||
|
}
|
||||||
|
if lc.HasAccessTransition() {
|
||||||
|
t.Fatal("HasAccessTransition = true with only a disabled rule")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccessTransitionCountsAsActiveRule(t *testing.T) {
|
||||||
|
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !lc.HasActiveRules("logs/2026") {
|
||||||
|
t.Fatal("access-only lifecycle rule was not active for its prefix")
|
||||||
|
}
|
||||||
|
if lc.HasActiveRules("data/") {
|
||||||
|
t.Fatal("access rule was active outside its prefix")
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -26,6 +26,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/google/uuid"
|
"github.com/google/uuid"
|
||||||
"github.com/minio/minio/internal/bucket/object/lock"
|
"github.com/minio/minio/internal/bucket/object/lock"
|
||||||
"github.com/minio/minio/internal/bucket/replication"
|
"github.com/minio/minio/internal/bucket/replication"
|
||||||
@@ -101,11 +102,56 @@ func (a Action) Delete() bool {
|
|||||||
|
|
||||||
// Lifecycle - Configuration for bucket lifecycle.
|
// Lifecycle - Configuration for bucket lifecycle.
|
||||||
type Lifecycle struct {
|
type Lifecycle struct {
|
||||||
XMLName xml.Name `xml:"LifecycleConfiguration"`
|
XMLName xml.Name `xml:"LifecycleConfiguration"`
|
||||||
|
// AccessTierQuota caps how many bytes of this bucket access-based ILM
|
||||||
|
// may keep on the fastest pool, e.g. "500GiB". Empty means unlimited.
|
||||||
|
// It is bucket-wide rather than per-rule so it cannot be declared
|
||||||
|
// inconsistently by two rules matching the same object.
|
||||||
|
AccessTierQuota string `xml:"AccessTierQuota,omitempty"`
|
||||||
Rules []Rule `xml:"Rule"`
|
Rules []Rule `xml:"Rule"`
|
||||||
ExpiryUpdatedAt *time.Time `xml:"ExpiryUpdatedAt,omitempty"`
|
ExpiryUpdatedAt *time.Time `xml:"ExpiryUpdatedAt,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HasAccessTransition returns 'true' if any enabled rule carries a usable
|
||||||
|
// AccessTransition. Used as a cheap per-bucket gate before consulting the
|
||||||
|
// access tracker.
|
||||||
|
func (lc Lifecycle) HasAccessTransition() bool {
|
||||||
|
for _, rule := range lc.Rules {
|
||||||
|
if rule.Status == Disabled {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if !rule.AccessTransition.IsNull() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessRule returns the first enabled rule matching obj that carries a usable
|
||||||
|
// AccessTransition, along with its rule ID.
|
||||||
|
func (lc Lifecycle) AccessRule(obj ObjectOpts) (AccessTransition, string, bool) {
|
||||||
|
for _, rule := range lc.FilterRules(obj) {
|
||||||
|
if !rule.AccessTransition.IsNull() {
|
||||||
|
return rule.AccessTransition, rule.ID, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return AccessTransition{}, "", false
|
||||||
|
}
|
||||||
|
|
||||||
|
// AccessQuotaBytes returns the bucket-wide fast-pool byte cap, 0 meaning
|
||||||
|
// unlimited. The value is validated at PUT time, so a parse failure here is
|
||||||
|
// treated as unlimited rather than as an error.
|
||||||
|
func (lc Lifecycle) AccessQuotaBytes() uint64 {
|
||||||
|
if lc.AccessTierQuota == "" {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
sz, err := humanize.ParseBytes(lc.AccessTierQuota)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return sz
|
||||||
|
}
|
||||||
|
|
||||||
// HasTransition returns 'true' if lifecycle document has Transition enabled.
|
// HasTransition returns 'true' if lifecycle document has Transition enabled.
|
||||||
func (lc Lifecycle) HasTransition() bool {
|
func (lc Lifecycle) HasTransition() bool {
|
||||||
for _, rule := range lc.Rules {
|
for _, rule := range lc.Rules {
|
||||||
@@ -158,6 +204,12 @@ func (lc *Lifecycle) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err e
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
lc.ExpiryUpdatedAt = &t
|
lc.ExpiryUpdatedAt = &t
|
||||||
|
case "AccessTierQuota":
|
||||||
|
var q string
|
||||||
|
if err = d.DecodeElement(&q, &se); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
lc.AccessTierQuota = q
|
||||||
default:
|
default:
|
||||||
return xml.UnmarshalError(fmt.Sprintf("expected element type <Rule> but have <%s>", se.Name.Local))
|
return xml.UnmarshalError(fmt.Sprintf("expected element type <Rule> but have <%s>", se.Name.Local))
|
||||||
}
|
}
|
||||||
@@ -208,6 +260,9 @@ func (lc Lifecycle) HasActiveRules(prefix string) bool {
|
|||||||
if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
|
if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
|
if !rule.AccessTransition.IsNull() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@@ -248,6 +303,12 @@ func (lc Lifecycle) Validate(lr lock.Retention) error {
|
|||||||
return errLifecycleNoRule
|
return errLifecycleNoRule
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if lc.AccessTierQuota != "" {
|
||||||
|
if _, err := humanize.ParseBytes(lc.AccessTierQuota); err != nil {
|
||||||
|
return errAccessInvalidQuotaSize
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Validate all the rules in the lifecycle config
|
// Validate all the rules in the lifecycle config
|
||||||
for _, r := range lc.Rules {
|
for _, r := range lc.Rules {
|
||||||
if err := r.Validate(); err != nil {
|
if err := r.Validate(); err != nil {
|
||||||
|
|||||||
@@ -41,6 +41,7 @@ type Rule struct {
|
|||||||
Expiration Expiration `xml:"Expiration,omitempty"`
|
Expiration Expiration `xml:"Expiration,omitempty"`
|
||||||
Transition Transition `xml:"Transition,omitempty"`
|
Transition Transition `xml:"Transition,omitempty"`
|
||||||
DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty"`
|
DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty"`
|
||||||
|
AccessTransition AccessTransition `xml:"AccessTransition,omitempty"`
|
||||||
// FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
// FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
|
||||||
NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
|
||||||
NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
|
NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
|
||||||
@@ -171,10 +172,13 @@ func (r Rule) Validate() error {
|
|||||||
if err := r.validateNoncurrentTransition(); err != nil {
|
if err := r.validateNoncurrentTransition(); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
if err := r.AccessTransition.Validate(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if (!r.Filter.Tag.IsEmpty() || len(r.Filter.And.Tags) != 0) && !r.DelMarkerExpiration.Empty() {
|
if (!r.Filter.Tag.IsEmpty() || len(r.Filter.And.Tags) != 0) && !r.DelMarkerExpiration.Empty() {
|
||||||
return errInvalidRuleDelMarkerExpiration
|
return errInvalidRuleDelMarkerExpiration
|
||||||
}
|
}
|
||||||
if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() {
|
if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() && !r.AccessTransition.set {
|
||||||
return errXMLNotWellFormed
|
return errXMLNotWellFormed
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -22,10 +22,43 @@ import "github.com/minio/minio/internal/config"
|
|||||||
const (
|
const (
|
||||||
transitionWorkers = "transition_workers"
|
transitionWorkers = "transition_workers"
|
||||||
expirationWorkers = "expiration_workers"
|
expirationWorkers = "expiration_workers"
|
||||||
|
|
||||||
|
accessTiering = "access_tiering"
|
||||||
|
accessPools = "access_pools"
|
||||||
|
accessMaxSize = "access_max_size"
|
||||||
|
accessPromoteWatermark = "access_promote_watermark"
|
||||||
|
accessBinWidth = "access_bin_width"
|
||||||
|
accessBins = "access_bins"
|
||||||
|
accessFlush = "access_flush"
|
||||||
|
accessMinResidency = "access_min_residency"
|
||||||
|
accessWorkers = "access_workers"
|
||||||
|
accessMaxTracked = "access_max_tracked"
|
||||||
|
|
||||||
// EnvILMTransitionWorkers env variable to configure number of transition workers
|
// EnvILMTransitionWorkers env variable to configure number of transition workers
|
||||||
EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS"
|
EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS"
|
||||||
// EnvILMExpirationWorkers env variable to configure number of expiration workers
|
// EnvILMExpirationWorkers env variable to configure number of expiration workers
|
||||||
EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS"
|
EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS"
|
||||||
|
|
||||||
|
// EnvILMAccessTiering env variable to enable access based tiering
|
||||||
|
EnvILMAccessTiering = "MINIO_ILM_ACCESS_TIERING"
|
||||||
|
// EnvILMAccessPools env variable listing pool indices hottest first
|
||||||
|
EnvILMAccessPools = "MINIO_ILM_ACCESS_POOLS"
|
||||||
|
// EnvILMAccessMaxSize env variable capping bytes held on the hottest pool
|
||||||
|
EnvILMAccessMaxSize = "MINIO_ILM_ACCESS_MAX_SIZE"
|
||||||
|
// EnvILMAccessPromoteWatermark env variable for the hottest pool fill limit
|
||||||
|
EnvILMAccessPromoteWatermark = "MINIO_ILM_ACCESS_PROMOTE_WATERMARK"
|
||||||
|
// EnvILMAccessBinWidth env variable for the access counter resolution
|
||||||
|
EnvILMAccessBinWidth = "MINIO_ILM_ACCESS_BIN_WIDTH"
|
||||||
|
// EnvILMAccessBins env variable for the number of access counter bins
|
||||||
|
EnvILMAccessBins = "MINIO_ILM_ACCESS_BINS"
|
||||||
|
// EnvILMAccessFlush env variable for the counter publish and sweep interval
|
||||||
|
EnvILMAccessFlush = "MINIO_ILM_ACCESS_FLUSH"
|
||||||
|
// EnvILMAccessMinResidency env variable for the anti-thrash floor
|
||||||
|
EnvILMAccessMinResidency = "MINIO_ILM_ACCESS_MIN_RESIDENCY"
|
||||||
|
// EnvILMAccessWorkers env variable to configure number of access tiering workers
|
||||||
|
EnvILMAccessWorkers = "MINIO_ILM_ACCESS_WORKERS"
|
||||||
|
// EnvILMAccessMaxTracked env variable capping tracked objects per node
|
||||||
|
EnvILMAccessMaxTracked = "MINIO_ILM_ACCESS_MAX_TRACKED"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -48,5 +81,65 @@ var (
|
|||||||
Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers),
|
Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers),
|
||||||
Optional: true,
|
Optional: true,
|
||||||
},
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessTiering,
|
||||||
|
Type: "on|off",
|
||||||
|
Description: `move objects between fast and slow server pools based on how often they are read` + defaultHelpPostfix(accessTiering),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessPools,
|
||||||
|
Type: "string",
|
||||||
|
Description: `server pool indices for access tiering, hottest first, e.g. "0,1"` + defaultHelpPostfix(accessPools),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessMaxSize,
|
||||||
|
Type: "string",
|
||||||
|
Description: `cap total bytes access tiering keeps on the fastest pool, e.g. "2TiB", 0 for unlimited` + defaultHelpPostfix(accessMaxSize),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessPromoteWatermark,
|
||||||
|
Type: "number",
|
||||||
|
Description: `stop promoting once the fastest pool is this percent full` + defaultHelpPostfix(accessPromoteWatermark),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessBinWidth,
|
||||||
|
Type: "duration",
|
||||||
|
Description: `resolution of the access counter` + defaultHelpPostfix(accessBinWidth),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessBins,
|
||||||
|
Type: "number",
|
||||||
|
Description: `number of access counter bins; bins times bin width caps the rule Window` + defaultHelpPostfix(accessBins),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessFlush,
|
||||||
|
Type: "duration",
|
||||||
|
Description: `how often access counters are published and a promotion sweep runs` + defaultHelpPostfix(accessFlush),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessMinResidency,
|
||||||
|
Type: "duration",
|
||||||
|
Description: `minimum time an object stays on a pool after access tiering moved it` + defaultHelpPostfix(accessMinResidency),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessWorkers,
|
||||||
|
Type: "number",
|
||||||
|
Description: `set the number of access tiering workers` + defaultHelpPostfix(accessWorkers),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
|
config.HelpKV{
|
||||||
|
Key: accessMaxTracked,
|
||||||
|
Type: "number",
|
||||||
|
Description: `maximum number of objects each node keeps access counters for` + defaultHelpPostfix(accessMaxTracked),
|
||||||
|
Optional: true,
|
||||||
|
},
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -18,12 +18,29 @@
|
|||||||
package ilm
|
package ilm
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"errors"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/dustin/go-humanize"
|
||||||
"github.com/minio/minio/internal/config"
|
"github.com/minio/minio/internal/config"
|
||||||
"github.com/minio/pkg/v3/env"
|
"github.com/minio/pkg/v3/env"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Errors returned when the access tiering configuration is unusable.
|
||||||
|
var (
|
||||||
|
ErrAccessPoolsInvalid = errors.New("ilm access_pools must be a comma separated list of distinct pool indices, hottest first, e.g. \"0,1\"")
|
||||||
|
ErrAccessPoolsTooFew = errors.New("ilm access_pools needs at least two pools to move objects between")
|
||||||
|
ErrAccessWatermarkInvalid = errors.New("ilm access_promote_watermark must be between 1 and 100")
|
||||||
|
ErrAccessBinsInvalid = errors.New("ilm access_bins must be between 2 and 64")
|
||||||
|
ErrAccessBinWidthInvalid = errors.New("ilm access_bin_width must be at least 1s")
|
||||||
|
ErrAccessFlushInvalid = errors.New("ilm access_flush must be at least 1s")
|
||||||
|
ErrAccessResidencyInvalid = errors.New("ilm access_min_residency cannot be negative")
|
||||||
|
ErrAccessWorkersInvalid = errors.New("ilm access_workers must be a positive integer")
|
||||||
|
ErrAccessTrackedInvalid = errors.New("ilm access_max_tracked must be a positive integer")
|
||||||
|
)
|
||||||
|
|
||||||
// DefaultKVS default configuration values for ILM subsystem
|
// DefaultKVS default configuration values for ILM subsystem
|
||||||
var DefaultKVS = config.KVS{
|
var DefaultKVS = config.KVS{
|
||||||
config.KV{
|
config.KV{
|
||||||
@@ -34,12 +51,124 @@ var DefaultKVS = config.KVS{
|
|||||||
Key: expirationWorkers,
|
Key: expirationWorkers,
|
||||||
Value: "100",
|
Value: "100",
|
||||||
},
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessTiering,
|
||||||
|
Value: config.EnableOff,
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessPools,
|
||||||
|
Value: "",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessMaxSize,
|
||||||
|
Value: "0",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessPromoteWatermark,
|
||||||
|
Value: "85",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessBinWidth,
|
||||||
|
Value: "1m",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessBins,
|
||||||
|
Value: "12",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessFlush,
|
||||||
|
Value: "1m",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessMinResidency,
|
||||||
|
Value: "24h",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessWorkers,
|
||||||
|
Value: "10",
|
||||||
|
},
|
||||||
|
config.KV{
|
||||||
|
Key: accessMaxTracked,
|
||||||
|
Value: "1000000",
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
// Config represents the different configuration values for ILM subsystem
|
// Config represents the different configuration values for ILM subsystem
|
||||||
type Config struct {
|
type Config struct {
|
||||||
TransitionWorkers int
|
TransitionWorkers int
|
||||||
ExpirationWorkers int
|
ExpirationWorkers int
|
||||||
|
|
||||||
|
// AccessTiering enables access-frequency driven relocation of objects
|
||||||
|
// between server pools. Off by default: it moves data.
|
||||||
|
AccessTiering bool
|
||||||
|
// AccessPools lists pool indices hottest first, e.g. []int{0, 1}.
|
||||||
|
AccessPools []int
|
||||||
|
// AccessMaxSize caps total bytes held on the hottest pool, 0 == unlimited.
|
||||||
|
AccessMaxSize uint64
|
||||||
|
// AccessPromoteWatermark stops promotion once the hottest pool is this
|
||||||
|
// percentage full.
|
||||||
|
AccessPromoteWatermark int
|
||||||
|
// AccessBinWidth and AccessBins size the rolling hit counter. Their
|
||||||
|
// product is the longest rule Window that can be evaluated.
|
||||||
|
AccessBinWidth time.Duration
|
||||||
|
AccessBins int
|
||||||
|
// AccessFlush is how often each node publishes its counters and the
|
||||||
|
// leader merges them and runs a promotion sweep.
|
||||||
|
AccessFlush time.Duration
|
||||||
|
// AccessMinResidency is the minimum time an object stays put after a
|
||||||
|
// move, regardless of what the counters say.
|
||||||
|
AccessMinResidency time.Duration
|
||||||
|
AccessWorkers int
|
||||||
|
// AccessMaxTracked caps how many objects each node keeps counters for.
|
||||||
|
AccessMaxTracked int
|
||||||
|
}
|
||||||
|
|
||||||
|
// HotPool returns the index of the hottest configured pool and whether access
|
||||||
|
// tiering is usable at all.
|
||||||
|
func (c Config) HotPool() (int, bool) {
|
||||||
|
if !c.AccessTiering || len(c.AccessPools) < 2 {
|
||||||
|
return -1, false
|
||||||
|
}
|
||||||
|
return c.AccessPools[0], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// ColdPool returns the index of the coldest configured pool, i.e. where
|
||||||
|
// demoted objects go.
|
||||||
|
func (c Config) ColdPool() (int, bool) {
|
||||||
|
if !c.AccessTiering || len(c.AccessPools) < 2 {
|
||||||
|
return -1, false
|
||||||
|
}
|
||||||
|
return c.AccessPools[len(c.AccessPools)-1], true
|
||||||
|
}
|
||||||
|
|
||||||
|
// HistoryWindow is the longest window the rolling counter can answer for.
|
||||||
|
func (c Config) HistoryWindow() time.Duration {
|
||||||
|
return time.Duration(c.AccessBins) * c.AccessBinWidth
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseAccessPools parses "0,1" into []int{0, 1}, rejecting duplicates and
|
||||||
|
// negative indices. An empty string yields no pools, which disables the
|
||||||
|
// feature rather than erroring - operators enable the switch before they
|
||||||
|
// configure the topology.
|
||||||
|
func parseAccessPools(s string) ([]int, error) {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
var pools []int
|
||||||
|
seen := make(map[int]struct{})
|
||||||
|
for _, f := range strings.Split(s, ",") {
|
||||||
|
idx, err := strconv.Atoi(strings.TrimSpace(f))
|
||||||
|
if err != nil || idx < 0 {
|
||||||
|
return nil, ErrAccessPoolsInvalid
|
||||||
|
}
|
||||||
|
if _, dup := seen[idx]; dup {
|
||||||
|
return nil, ErrAccessPoolsInvalid
|
||||||
|
}
|
||||||
|
seen[idx] = struct{}{}
|
||||||
|
pools = append(pools, idx)
|
||||||
|
}
|
||||||
|
return pools, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// LookupConfig - lookup ilm config and override with valid environment settings if any.
|
// LookupConfig - lookup ilm config and override with valid environment settings if any.
|
||||||
@@ -65,5 +194,79 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
|
|||||||
|
|
||||||
cfg.TransitionWorkers = tw
|
cfg.TransitionWorkers = tw
|
||||||
cfg.ExpirationWorkers = ew
|
cfg.ExpirationWorkers = ew
|
||||||
|
|
||||||
|
if err := cfg.lookupAccess(kvs); err != nil {
|
||||||
|
return cfg, err
|
||||||
|
}
|
||||||
return cfg, nil
|
return cfg, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *Config) lookupAccess(kvs config.KVS) (err error) {
|
||||||
|
c.AccessTiering, err = config.ParseBool(env.Get(EnvILMAccessTiering, kvs.GetWithDefault(accessTiering, DefaultKVS)))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.AccessPools, err = parseAccessPools(env.Get(EnvILMAccessPools, kvs.GetWithDefault(accessPools, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
// Enabling the feature without a usable topology is a configuration
|
||||||
|
// error worth surfacing at set time rather than silently doing nothing.
|
||||||
|
if c.AccessTiering && len(c.AccessPools) < 2 {
|
||||||
|
return ErrAccessPoolsTooFew
|
||||||
|
}
|
||||||
|
|
||||||
|
maxSize := env.Get(EnvILMAccessMaxSize, kvs.GetWithDefault(accessMaxSize, DefaultKVS))
|
||||||
|
if maxSize == "" || maxSize == "0" {
|
||||||
|
c.AccessMaxSize = 0
|
||||||
|
} else if c.AccessMaxSize, err = humanize.ParseBytes(maxSize); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.AccessPromoteWatermark, err = strconv.Atoi(env.Get(EnvILMAccessPromoteWatermark, kvs.GetWithDefault(accessPromoteWatermark, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.AccessPromoteWatermark < 1 || c.AccessPromoteWatermark > 100 {
|
||||||
|
return ErrAccessWatermarkInvalid
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.AccessBinWidth, err = time.ParseDuration(env.Get(EnvILMAccessBinWidth, kvs.GetWithDefault(accessBinWidth, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.AccessBinWidth < time.Second {
|
||||||
|
return ErrAccessBinWidthInvalid
|
||||||
|
}
|
||||||
|
if c.AccessBins, err = strconv.Atoi(env.Get(EnvILMAccessBins, kvs.GetWithDefault(accessBins, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.AccessBins < 2 || c.AccessBins > 64 {
|
||||||
|
return ErrAccessBinsInvalid
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.AccessFlush, err = time.ParseDuration(env.Get(EnvILMAccessFlush, kvs.GetWithDefault(accessFlush, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.AccessFlush < time.Second {
|
||||||
|
return ErrAccessFlushInvalid
|
||||||
|
}
|
||||||
|
if c.AccessMinResidency, err = time.ParseDuration(env.Get(EnvILMAccessMinResidency, kvs.GetWithDefault(accessMinResidency, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.AccessMinResidency < 0 {
|
||||||
|
return ErrAccessResidencyInvalid
|
||||||
|
}
|
||||||
|
|
||||||
|
if c.AccessWorkers, err = strconv.Atoi(env.Get(EnvILMAccessWorkers, kvs.GetWithDefault(accessWorkers, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.AccessWorkers < 1 {
|
||||||
|
return ErrAccessWorkersInvalid
|
||||||
|
}
|
||||||
|
if c.AccessMaxTracked, err = strconv.Atoi(env.Get(EnvILMAccessMaxTracked, kvs.GetWithDefault(accessMaxTracked, DefaultKVS))); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.AccessMaxTracked < 1 {
|
||||||
|
return ErrAccessTrackedInvalid
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,109 @@
|
|||||||
|
// Copyright (c) 2015-2026 MinIO, Inc.
|
||||||
|
|
||||||
|
package ilm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/minio/minio/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func clearAccessEnv(t *testing.T) {
|
||||||
|
t.Helper()
|
||||||
|
for _, name := range []string{
|
||||||
|
EnvILMAccessTiering, EnvILMAccessPools, EnvILMAccessMaxSize,
|
||||||
|
EnvILMAccessPromoteWatermark, EnvILMAccessBinWidth, EnvILMAccessBins,
|
||||||
|
EnvILMAccessFlush, EnvILMAccessMinResidency, EnvILMAccessWorkers,
|
||||||
|
EnvILMAccessMaxTracked,
|
||||||
|
} {
|
||||||
|
// The env helper treats an empty value as unset. t.Setenv restores the
|
||||||
|
// caller's exact value automatically when the test finishes.
|
||||||
|
t.Setenv(name, "")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLookupAccessDefaults(t *testing.T) {
|
||||||
|
clearAccessEnv(t)
|
||||||
|
cfg, err := LookupConfig(DefaultKVS.Clone())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if cfg.AccessTiering {
|
||||||
|
t.Fatal("access tiering must default off")
|
||||||
|
}
|
||||||
|
if cfg.AccessBinWidth != time.Minute || cfg.AccessBins != 12 || cfg.AccessFlush != time.Minute {
|
||||||
|
t.Fatalf("unexpected counter defaults: width=%s bins=%d flush=%s", cfg.AccessBinWidth, cfg.AccessBins, cfg.AccessFlush)
|
||||||
|
}
|
||||||
|
if cfg.AccessWorkers != 10 || cfg.AccessMaxTracked != 1000000 {
|
||||||
|
t.Fatalf("unexpected worker/map defaults: %d/%d", cfg.AccessWorkers, cfg.AccessMaxTracked)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLookupAccessEnabled(t *testing.T) {
|
||||||
|
clearAccessEnv(t)
|
||||||
|
kvs := DefaultKVS.Clone()
|
||||||
|
kvs.Set(accessTiering, config.EnableOn)
|
||||||
|
kvs.Set(accessPools, "2, 0, 1")
|
||||||
|
kvs.Set(accessMaxSize, "2GiB")
|
||||||
|
kvs.Set(accessPromoteWatermark, "90")
|
||||||
|
kvs.Set(accessBinWidth, "30s")
|
||||||
|
kvs.Set(accessBins, "20")
|
||||||
|
kvs.Set(accessFlush, "15s")
|
||||||
|
kvs.Set(accessMinResidency, "2h")
|
||||||
|
kvs.Set(accessWorkers, "7")
|
||||||
|
kvs.Set(accessMaxTracked, "1234")
|
||||||
|
|
||||||
|
cfg, err := LookupConfig(kvs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !cfg.AccessTiering || !reflect.DeepEqual(cfg.AccessPools, []int{2, 0, 1}) {
|
||||||
|
t.Fatalf("unexpected topology: enabled=%v pools=%v", cfg.AccessTiering, cfg.AccessPools)
|
||||||
|
}
|
||||||
|
if cfg.AccessMaxSize != 2<<30 || cfg.HistoryWindow() != 10*time.Minute {
|
||||||
|
t.Fatalf("unexpected size/history: %d/%s", cfg.AccessMaxSize, cfg.HistoryWindow())
|
||||||
|
}
|
||||||
|
if hot, ok := cfg.HotPool(); !ok || hot != 2 {
|
||||||
|
t.Fatalf("HotPool = %d/%v", hot, ok)
|
||||||
|
}
|
||||||
|
if cold, ok := cfg.ColdPool(); !ok || cold != 1 {
|
||||||
|
t.Fatalf("ColdPool = %d/%v", cold, ok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLookupAccessRejectsInvalidValues(t *testing.T) {
|
||||||
|
clearAccessEnv(t)
|
||||||
|
tests := []struct {
|
||||||
|
key, value string
|
||||||
|
want error
|
||||||
|
}{
|
||||||
|
{accessPools, "0,0", ErrAccessPoolsInvalid},
|
||||||
|
{accessPromoteWatermark, "0", ErrAccessWatermarkInvalid},
|
||||||
|
{accessBinWidth, "500ms", ErrAccessBinWidthInvalid},
|
||||||
|
{accessBins, "1", ErrAccessBinsInvalid},
|
||||||
|
{accessFlush, "0s", ErrAccessFlushInvalid},
|
||||||
|
{accessMinResidency, "-1s", ErrAccessResidencyInvalid},
|
||||||
|
{accessWorkers, "0", ErrAccessWorkersInvalid},
|
||||||
|
{accessMaxTracked, "0", ErrAccessTrackedInvalid},
|
||||||
|
}
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.key, func(t *testing.T) {
|
||||||
|
kvs := DefaultKVS.Clone()
|
||||||
|
kvs.Set(tc.key, tc.value)
|
||||||
|
_, err := LookupConfig(kvs)
|
||||||
|
if !errors.Is(err, tc.want) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
kvs := DefaultKVS.Clone()
|
||||||
|
kvs.Set(accessTiering, config.EnableOn)
|
||||||
|
kvs.Set(accessPools, "0")
|
||||||
|
if _, err := LookupConfig(kvs); !errors.Is(err, ErrAccessPoolsTooFew) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, ErrAccessPoolsTooFew)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user