feat(ilm): relocate hot objects across server pools by GET frequency

Keep NVMe/HDD pool pairs useful without remote tiering: promote objects that
are read often, demote previously moved objects once they go idle, and leave
the feature off until operators set a two-pool topology.

Signed-off-by: mr javad seydi <seydi.birjand@gmail.com>
This commit is contained in:
mr javad seydi
2026-08-15 13:39:02 +03:30
parent 100e2e57a7
commit 7a060cab1e
31 changed files with 5042 additions and 42 deletions
@@ -0,0 +1,129 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"encoding/xml"
"time"
)
var (
errAccessInvalidDuration = Errorf("Window and DemoteAfterIdle must be valid Go durations, e.g. 10m or 24h")
errAccessInvalidWindow = Errorf("Window must be a positive duration with AccessTransition")
errAccessInvalidPromote = Errorf("PromoteAfterAccesses must be a positive integer with AccessTransition")
errAccessInvalidDemote = Errorf("DemoteAfterAccesses must be smaller than PromoteAfterAccesses and 0 or greater")
errAccessInvalidIdle = Errorf("DemoteAfterIdle must be a positive duration no shorter than Window")
errAccessInvalidQuotaSize = Errorf("AccessTierQuota must be a valid size, e.g. 500GiB")
)
// Duration is a time.Duration that marshals to and from an XML element
// holding a Go duration string, e.g. <Window>10m</Window>.
type Duration time.Duration
// UnmarshalXML parses a duration string such as "10m" or "24h".
func (d *Duration) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error {
var s string
if err := dec.DecodeElement(&s, &start); err != nil {
return err
}
dur, err := time.ParseDuration(s)
if err != nil {
return errAccessInvalidDuration
}
*d = Duration(dur)
return nil
}
// MarshalXML encodes a non-zero duration, and nothing otherwise.
func (d Duration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
if d == 0 {
return nil
}
return enc.EncodeElement(time.Duration(d).String(), start)
}
// D returns the value as a time.Duration.
func (d Duration) D() time.Duration {
return time.Duration(d)
}
// AccessTransition is a Silo extension to the S3 lifecycle rule. It relocates
// an object between server pools based on how often it is read, rather than on
// its age: an object read at least PromoteAfterAccesses times within Window
// moves to the fastest configured pool, and moves back once it has been idle
// for DemoteAfterIdle and its windowed hit count has fallen to
// DemoteAfterAccesses or below.
//
// The gap between the two thresholds, together with DemoteAfterIdle and the
// server-side access_min_residency, is what keeps an object from oscillating
// between pools.
type AccessTransition struct {
XMLName xml.Name `xml:"AccessTransition"`
Window Duration `xml:"Window,omitempty"`
PromoteAfterAccesses int `xml:"PromoteAfterAccesses,omitempty"`
DemoteAfterAccesses int `xml:"DemoteAfterAccesses,omitempty"`
DemoteAfterIdle Duration `xml:"DemoteAfterIdle,omitempty"`
set bool
}
// IsNull returns true if no usable access transition is configured.
func (a AccessTransition) IsNull() bool {
return !a.set || a.PromoteAfterAccesses <= 0
}
// MarshalXML encodes an AccessTransition element, and nothing if unset.
func (a AccessTransition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
if !a.set {
return nil
}
type accessTransitionWrapper AccessTransition
return enc.EncodeElement(accessTransitionWrapper(a), start)
}
// UnmarshalXML decodes an AccessTransition element.
func (a *AccessTransition) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error {
type accessTransitionWrapper AccessTransition
var atw accessTransitionWrapper
if err := dec.DecodeElement(&atw, &start); err != nil {
return err
}
*a = AccessTransition(atw)
a.set = true
return nil
}
// Validate checks the AccessTransition element.
func (a AccessTransition) Validate() error {
if !a.set {
return nil
}
if a.Window <= 0 {
return errAccessInvalidWindow
}
if a.PromoteAfterAccesses <= 0 {
return errAccessInvalidPromote
}
if a.DemoteAfterAccesses < 0 || a.DemoteAfterAccesses >= a.PromoteAfterAccesses {
return errAccessInvalidDemote
}
if a.DemoteAfterIdle <= 0 || a.DemoteAfterIdle < a.Window {
return errAccessInvalidIdle
}
return nil
}
@@ -0,0 +1,219 @@
// Copyright (c) 2015-2026 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package lifecycle
import (
"bytes"
"encoding/xml"
"strings"
"testing"
"time"
"github.com/minio/minio/internal/bucket/object/lock"
)
const accessTieringXML = `<LifecycleConfiguration>
<AccessTierQuota>500GiB</AccessTierQuota>
<Rule>
<ID>hot-logs</ID>
<Status>Enabled</Status>
<Filter><And><Prefix>logs/</Prefix><ObjectSizeGreaterThan>65536</ObjectSizeGreaterThan></And></Filter>
<AccessTransition>
<Window>10m</Window>
<PromoteAfterAccesses>100</PromoteAfterAccesses>
<DemoteAfterAccesses>5</DemoteAfterAccesses>
<DemoteAfterIdle>24h</DemoteAfterIdle>
</AccessTransition>
</Rule>
</LifecycleConfiguration>`
func TestAccessTransitionParse(t *testing.T) {
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
if err != nil {
t.Fatalf("parse: %v", err)
}
if err := lc.Validate(lock.Retention{}); err != nil {
t.Fatalf("validate: %v", err)
}
if got := lc.AccessQuotaBytes(); got != 500*1024*1024*1024 {
t.Fatalf("quota = %d, want %d", got, 500*1024*1024*1024)
}
if !lc.HasAccessTransition() {
t.Fatal("HasAccessTransition = false, want true")
}
at := lc.Rules[0].AccessTransition
if at.Window.D() != 10*time.Minute {
t.Fatalf("window = %v, want 10m", at.Window.D())
}
if at.DemoteAfterIdle.D() != 24*time.Hour {
t.Fatalf("idle = %v, want 24h", at.DemoteAfterIdle.D())
}
if at.PromoteAfterAccesses != 100 || at.DemoteAfterAccesses != 5 {
t.Fatalf("thresholds = %d/%d, want 100/5", at.PromoteAfterAccesses, at.DemoteAfterAccesses)
}
}
// A round trip through Marshal must preserve both the rule element and the
// bucket-wide quota, since PutBucketLifecycle stores whatever we re-encode.
func TestAccessTransitionRoundTrip(t *testing.T) {
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
if err != nil {
t.Fatalf("parse: %v", err)
}
buf, err := xml.Marshal(lc)
if err != nil {
t.Fatalf("marshal: %v", err)
}
if !bytes.Contains(buf, []byte("<AccessTierQuota>500GiB</AccessTierQuota>")) {
t.Fatalf("quota lost in round trip: %s", buf)
}
got, err := ParseLifecycleConfig(bytes.NewReader(buf))
if err != nil {
t.Fatalf("reparse: %v", err)
}
if got.AccessQuotaBytes() != lc.AccessQuotaBytes() {
t.Fatalf("quota %d != %d", got.AccessQuotaBytes(), lc.AccessQuotaBytes())
}
if got.Rules[0].AccessTransition != lc.Rules[0].AccessTransition {
t.Fatalf("rule %+v != %+v", got.Rules[0].AccessTransition, lc.Rules[0].AccessTransition)
}
}
// A rule with no AccessTransition must not emit an empty element - otherwise
// every existing lifecycle config would change shape on rewrite.
func TestAccessTransitionUnsetNotMarshalled(t *testing.T) {
lc, err := ParseLifecycleConfig(strings.NewReader(`<LifecycleConfiguration><Rule>
<ID>old</ID><Status>Enabled</Status><Filter><Prefix>a/</Prefix></Filter>
<Expiration><Days>3</Days></Expiration></Rule></LifecycleConfiguration>`))
if err != nil {
t.Fatalf("parse: %v", err)
}
buf, err := xml.Marshal(lc)
if err != nil {
t.Fatalf("marshal: %v", err)
}
if bytes.Contains(buf, []byte("AccessTransition")) || bytes.Contains(buf, []byte("AccessTierQuota")) {
t.Fatalf("unset elements emitted: %s", buf)
}
if lc.HasAccessTransition() {
t.Fatal("HasAccessTransition = true for a plain expiry rule")
}
}
func TestAccessTransitionValidate(t *testing.T) {
tests := []struct {
name string
at AccessTransition
err error
}{
{"ok", AccessTransition{Window: Duration(10 * time.Minute), PromoteAfterAccesses: 100, DemoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Hour), set: true}, nil},
{"unset", AccessTransition{}, nil},
{"zero window", AccessTransition{PromoteAfterAccesses: 100, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidWindow},
{"zero promote", AccessTransition{Window: Duration(time.Minute), DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidPromote},
{"demote >= promote", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, DemoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidDemote},
{"negative demote", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, DemoteAfterAccesses: -1, DemoteAfterIdle: Duration(time.Hour), set: true}, errAccessInvalidDemote},
{"idle shorter than window", AccessTransition{Window: Duration(time.Hour), PromoteAfterAccesses: 5, DemoteAfterIdle: Duration(time.Minute), set: true}, errAccessInvalidIdle},
{"zero idle", AccessTransition{Window: Duration(time.Minute), PromoteAfterAccesses: 5, set: true}, errAccessInvalidIdle},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if err := tc.at.Validate(); err != tc.err {
t.Fatalf("err = %v, want %v", err, tc.err)
}
})
}
}
func TestAccessTierQuotaInvalid(t *testing.T) {
lc, err := ParseLifecycleConfig(strings.NewReader(`<LifecycleConfiguration>
<AccessTierQuota>not-a-size</AccessTierQuota>
<Rule><ID>r</ID><Status>Enabled</Status><Expiration><Days>3</Days></Expiration></Rule>
</LifecycleConfiguration>`))
if err != nil {
t.Fatalf("parse: %v", err)
}
if err := lc.Validate(lock.Retention{}); err != errAccessInvalidQuotaSize {
t.Fatalf("err = %v, want %v", err, errAccessInvalidQuotaSize)
}
// An invalid quota that somehow reached the evaluator means "unlimited",
// never "zero bytes allowed".
if got := lc.AccessQuotaBytes(); got != 0 {
t.Fatalf("quota = %d, want 0 (unlimited)", got)
}
}
func TestAccessTransitionBadDuration(t *testing.T) {
_, err := ParseLifecycleConfig(strings.NewReader(`<LifecycleConfiguration><Rule>
<ID>r</ID><Status>Enabled</Status>
<AccessTransition><Window>ten minutes</Window><PromoteAfterAccesses>1</PromoteAfterAccesses></AccessTransition>
</Rule></LifecycleConfiguration>`))
if err == nil {
t.Fatal("expected a parse error for a malformed duration")
}
}
// AccessRule must reuse the standard rule filtering: prefix, tags, size and
// Status all have to be honored.
func TestAccessRuleFiltering(t *testing.T) {
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
if err != nil {
t.Fatalf("parse: %v", err)
}
tests := []struct {
name string
obj ObjectOpts
want bool
}{
{"match", ObjectOpts{Name: "logs/a.log", Size: 1 << 20, IsLatest: true}, true},
{"wrong prefix", ObjectOpts{Name: "data/a.log", Size: 1 << 20, IsLatest: true}, false},
{"too small", ObjectOpts{Name: "logs/a.log", Size: 1024, IsLatest: true}, false},
{"no name", ObjectOpts{Size: 1 << 20}, false},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, id, ok := lc.AccessRule(tc.obj)
if ok != tc.want {
t.Fatalf("ok = %v, want %v", ok, tc.want)
}
if ok && id != "hot-logs" {
t.Fatalf("ruleID = %q, want hot-logs", id)
}
})
}
lc.Rules[0].Status = Disabled
if _, _, ok := lc.AccessRule(ObjectOpts{Name: "logs/a.log", Size: 1 << 20, IsLatest: true}); ok {
t.Fatal("disabled rule still matched")
}
if lc.HasAccessTransition() {
t.Fatal("HasAccessTransition = true with only a disabled rule")
}
}
func TestAccessTransitionCountsAsActiveRule(t *testing.T) {
lc, err := ParseLifecycleConfig(strings.NewReader(accessTieringXML))
if err != nil {
t.Fatal(err)
}
if !lc.HasActiveRules("logs/2026") {
t.Fatal("access-only lifecycle rule was not active for its prefix")
}
if lc.HasActiveRules("data/") {
t.Fatal("access rule was active outside its prefix")
}
}
+62 -1
View File
@@ -26,6 +26,7 @@ import (
"strings"
"time"
"github.com/dustin/go-humanize"
"github.com/google/uuid"
"github.com/minio/minio/internal/bucket/object/lock"
"github.com/minio/minio/internal/bucket/replication"
@@ -101,11 +102,56 @@ func (a Action) Delete() bool {
// Lifecycle - Configuration for bucket lifecycle.
type Lifecycle struct {
XMLName xml.Name `xml:"LifecycleConfiguration"`
XMLName xml.Name `xml:"LifecycleConfiguration"`
// AccessTierQuota caps how many bytes of this bucket access-based ILM
// may keep on the fastest pool, e.g. "500GiB". Empty means unlimited.
// It is bucket-wide rather than per-rule so it cannot be declared
// inconsistently by two rules matching the same object.
AccessTierQuota string `xml:"AccessTierQuota,omitempty"`
Rules []Rule `xml:"Rule"`
ExpiryUpdatedAt *time.Time `xml:"ExpiryUpdatedAt,omitempty"`
}
// HasAccessTransition returns 'true' if any enabled rule carries a usable
// AccessTransition. Used as a cheap per-bucket gate before consulting the
// access tracker.
func (lc Lifecycle) HasAccessTransition() bool {
for _, rule := range lc.Rules {
if rule.Status == Disabled {
continue
}
if !rule.AccessTransition.IsNull() {
return true
}
}
return false
}
// AccessRule returns the first enabled rule matching obj that carries a usable
// AccessTransition, along with its rule ID.
func (lc Lifecycle) AccessRule(obj ObjectOpts) (AccessTransition, string, bool) {
for _, rule := range lc.FilterRules(obj) {
if !rule.AccessTransition.IsNull() {
return rule.AccessTransition, rule.ID, true
}
}
return AccessTransition{}, "", false
}
// AccessQuotaBytes returns the bucket-wide fast-pool byte cap, 0 meaning
// unlimited. The value is validated at PUT time, so a parse failure here is
// treated as unlimited rather than as an error.
func (lc Lifecycle) AccessQuotaBytes() uint64 {
if lc.AccessTierQuota == "" {
return 0
}
sz, err := humanize.ParseBytes(lc.AccessTierQuota)
if err != nil {
return 0
}
return sz
}
// HasTransition returns 'true' if lifecycle document has Transition enabled.
func (lc Lifecycle) HasTransition() bool {
for _, rule := range lc.Rules {
@@ -158,6 +204,12 @@ func (lc *Lifecycle) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err e
return err
}
lc.ExpiryUpdatedAt = &t
case "AccessTierQuota":
var q string
if err = d.DecodeElement(&q, &se); err != nil {
return err
}
lc.AccessTierQuota = q
default:
return xml.UnmarshalError(fmt.Sprintf("expected element type <Rule> but have <%s>", se.Name.Local))
}
@@ -208,6 +260,9 @@ func (lc Lifecycle) HasActiveRules(prefix string) bool {
if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero.
return true
}
if !rule.AccessTransition.IsNull() {
return true
}
}
return false
}
@@ -248,6 +303,12 @@ func (lc Lifecycle) Validate(lr lock.Retention) error {
return errLifecycleNoRule
}
if lc.AccessTierQuota != "" {
if _, err := humanize.ParseBytes(lc.AccessTierQuota); err != nil {
return errAccessInvalidQuotaSize
}
}
// Validate all the rules in the lifecycle config
for _, r := range lc.Rules {
if err := r.Validate(); err != nil {
+5 -1
View File
@@ -41,6 +41,7 @@ type Rule struct {
Expiration Expiration `xml:"Expiration,omitempty"`
Transition Transition `xml:"Transition,omitempty"`
DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty"`
AccessTransition AccessTransition `xml:"AccessTransition,omitempty"`
// FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"`
NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"`
NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"`
@@ -171,10 +172,13 @@ func (r Rule) Validate() error {
if err := r.validateNoncurrentTransition(); err != nil {
return err
}
if err := r.AccessTransition.Validate(); err != nil {
return err
}
if (!r.Filter.Tag.IsEmpty() || len(r.Filter.And.Tags) != 0) && !r.DelMarkerExpiration.Empty() {
return errInvalidRuleDelMarkerExpiration
}
if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() {
if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() && !r.AccessTransition.set {
return errXMLNotWellFormed
}
return nil
+93
View File
@@ -22,10 +22,43 @@ import "github.com/minio/minio/internal/config"
const (
transitionWorkers = "transition_workers"
expirationWorkers = "expiration_workers"
accessTiering = "access_tiering"
accessPools = "access_pools"
accessMaxSize = "access_max_size"
accessPromoteWatermark = "access_promote_watermark"
accessBinWidth = "access_bin_width"
accessBins = "access_bins"
accessFlush = "access_flush"
accessMinResidency = "access_min_residency"
accessWorkers = "access_workers"
accessMaxTracked = "access_max_tracked"
// EnvILMTransitionWorkers env variable to configure number of transition workers
EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS"
// EnvILMExpirationWorkers env variable to configure number of expiration workers
EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS"
// EnvILMAccessTiering env variable to enable access based tiering
EnvILMAccessTiering = "MINIO_ILM_ACCESS_TIERING"
// EnvILMAccessPools env variable listing pool indices hottest first
EnvILMAccessPools = "MINIO_ILM_ACCESS_POOLS"
// EnvILMAccessMaxSize env variable capping bytes held on the hottest pool
EnvILMAccessMaxSize = "MINIO_ILM_ACCESS_MAX_SIZE"
// EnvILMAccessPromoteWatermark env variable for the hottest pool fill limit
EnvILMAccessPromoteWatermark = "MINIO_ILM_ACCESS_PROMOTE_WATERMARK"
// EnvILMAccessBinWidth env variable for the access counter resolution
EnvILMAccessBinWidth = "MINIO_ILM_ACCESS_BIN_WIDTH"
// EnvILMAccessBins env variable for the number of access counter bins
EnvILMAccessBins = "MINIO_ILM_ACCESS_BINS"
// EnvILMAccessFlush env variable for the counter publish and sweep interval
EnvILMAccessFlush = "MINIO_ILM_ACCESS_FLUSH"
// EnvILMAccessMinResidency env variable for the anti-thrash floor
EnvILMAccessMinResidency = "MINIO_ILM_ACCESS_MIN_RESIDENCY"
// EnvILMAccessWorkers env variable to configure number of access tiering workers
EnvILMAccessWorkers = "MINIO_ILM_ACCESS_WORKERS"
// EnvILMAccessMaxTracked env variable capping tracked objects per node
EnvILMAccessMaxTracked = "MINIO_ILM_ACCESS_MAX_TRACKED"
)
var (
@@ -48,5 +81,65 @@ var (
Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers),
Optional: true,
},
config.HelpKV{
Key: accessTiering,
Type: "on|off",
Description: `move objects between fast and slow server pools based on how often they are read` + defaultHelpPostfix(accessTiering),
Optional: true,
},
config.HelpKV{
Key: accessPools,
Type: "string",
Description: `server pool indices for access tiering, hottest first, e.g. "0,1"` + defaultHelpPostfix(accessPools),
Optional: true,
},
config.HelpKV{
Key: accessMaxSize,
Type: "string",
Description: `cap total bytes access tiering keeps on the fastest pool, e.g. "2TiB", 0 for unlimited` + defaultHelpPostfix(accessMaxSize),
Optional: true,
},
config.HelpKV{
Key: accessPromoteWatermark,
Type: "number",
Description: `stop promoting once the fastest pool is this percent full` + defaultHelpPostfix(accessPromoteWatermark),
Optional: true,
},
config.HelpKV{
Key: accessBinWidth,
Type: "duration",
Description: `resolution of the access counter` + defaultHelpPostfix(accessBinWidth),
Optional: true,
},
config.HelpKV{
Key: accessBins,
Type: "number",
Description: `number of access counter bins; bins times bin width caps the rule Window` + defaultHelpPostfix(accessBins),
Optional: true,
},
config.HelpKV{
Key: accessFlush,
Type: "duration",
Description: `how often access counters are published and a promotion sweep runs` + defaultHelpPostfix(accessFlush),
Optional: true,
},
config.HelpKV{
Key: accessMinResidency,
Type: "duration",
Description: `minimum time an object stays on a pool after access tiering moved it` + defaultHelpPostfix(accessMinResidency),
Optional: true,
},
config.HelpKV{
Key: accessWorkers,
Type: "number",
Description: `set the number of access tiering workers` + defaultHelpPostfix(accessWorkers),
Optional: true,
},
config.HelpKV{
Key: accessMaxTracked,
Type: "number",
Description: `maximum number of objects each node keeps access counters for` + defaultHelpPostfix(accessMaxTracked),
Optional: true,
},
}
)
+203
View File
@@ -18,12 +18,29 @@
package ilm
import (
"errors"
"strconv"
"strings"
"time"
"github.com/dustin/go-humanize"
"github.com/minio/minio/internal/config"
"github.com/minio/pkg/v3/env"
)
// Errors returned when the access tiering configuration is unusable.
var (
ErrAccessPoolsInvalid = errors.New("ilm access_pools must be a comma separated list of distinct pool indices, hottest first, e.g. \"0,1\"")
ErrAccessPoolsTooFew = errors.New("ilm access_pools needs at least two pools to move objects between")
ErrAccessWatermarkInvalid = errors.New("ilm access_promote_watermark must be between 1 and 100")
ErrAccessBinsInvalid = errors.New("ilm access_bins must be between 2 and 64")
ErrAccessBinWidthInvalid = errors.New("ilm access_bin_width must be at least 1s")
ErrAccessFlushInvalid = errors.New("ilm access_flush must be at least 1s")
ErrAccessResidencyInvalid = errors.New("ilm access_min_residency cannot be negative")
ErrAccessWorkersInvalid = errors.New("ilm access_workers must be a positive integer")
ErrAccessTrackedInvalid = errors.New("ilm access_max_tracked must be a positive integer")
)
// DefaultKVS default configuration values for ILM subsystem
var DefaultKVS = config.KVS{
config.KV{
@@ -34,12 +51,124 @@ var DefaultKVS = config.KVS{
Key: expirationWorkers,
Value: "100",
},
config.KV{
Key: accessTiering,
Value: config.EnableOff,
},
config.KV{
Key: accessPools,
Value: "",
},
config.KV{
Key: accessMaxSize,
Value: "0",
},
config.KV{
Key: accessPromoteWatermark,
Value: "85",
},
config.KV{
Key: accessBinWidth,
Value: "1m",
},
config.KV{
Key: accessBins,
Value: "12",
},
config.KV{
Key: accessFlush,
Value: "1m",
},
config.KV{
Key: accessMinResidency,
Value: "24h",
},
config.KV{
Key: accessWorkers,
Value: "10",
},
config.KV{
Key: accessMaxTracked,
Value: "1000000",
},
}
// Config represents the different configuration values for ILM subsystem
type Config struct {
TransitionWorkers int
ExpirationWorkers int
// AccessTiering enables access-frequency driven relocation of objects
// between server pools. Off by default: it moves data.
AccessTiering bool
// AccessPools lists pool indices hottest first, e.g. []int{0, 1}.
AccessPools []int
// AccessMaxSize caps total bytes held on the hottest pool, 0 == unlimited.
AccessMaxSize uint64
// AccessPromoteWatermark stops promotion once the hottest pool is this
// percentage full.
AccessPromoteWatermark int
// AccessBinWidth and AccessBins size the rolling hit counter. Their
// product is the longest rule Window that can be evaluated.
AccessBinWidth time.Duration
AccessBins int
// AccessFlush is how often each node publishes its counters and the
// leader merges them and runs a promotion sweep.
AccessFlush time.Duration
// AccessMinResidency is the minimum time an object stays put after a
// move, regardless of what the counters say.
AccessMinResidency time.Duration
AccessWorkers int
// AccessMaxTracked caps how many objects each node keeps counters for.
AccessMaxTracked int
}
// HotPool returns the index of the hottest configured pool and whether access
// tiering is usable at all.
func (c Config) HotPool() (int, bool) {
if !c.AccessTiering || len(c.AccessPools) < 2 {
return -1, false
}
return c.AccessPools[0], true
}
// ColdPool returns the index of the coldest configured pool, i.e. where
// demoted objects go.
func (c Config) ColdPool() (int, bool) {
if !c.AccessTiering || len(c.AccessPools) < 2 {
return -1, false
}
return c.AccessPools[len(c.AccessPools)-1], true
}
// HistoryWindow is the longest window the rolling counter can answer for.
func (c Config) HistoryWindow() time.Duration {
return time.Duration(c.AccessBins) * c.AccessBinWidth
}
// parseAccessPools parses "0,1" into []int{0, 1}, rejecting duplicates and
// negative indices. An empty string yields no pools, which disables the
// feature rather than erroring - operators enable the switch before they
// configure the topology.
func parseAccessPools(s string) ([]int, error) {
s = strings.TrimSpace(s)
if s == "" {
return nil, nil
}
var pools []int
seen := make(map[int]struct{})
for _, f := range strings.Split(s, ",") {
idx, err := strconv.Atoi(strings.TrimSpace(f))
if err != nil || idx < 0 {
return nil, ErrAccessPoolsInvalid
}
if _, dup := seen[idx]; dup {
return nil, ErrAccessPoolsInvalid
}
seen[idx] = struct{}{}
pools = append(pools, idx)
}
return pools, nil
}
// LookupConfig - lookup ilm config and override with valid environment settings if any.
@@ -65,5 +194,79 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
cfg.TransitionWorkers = tw
cfg.ExpirationWorkers = ew
if err := cfg.lookupAccess(kvs); err != nil {
return cfg, err
}
return cfg, nil
}
func (c *Config) lookupAccess(kvs config.KVS) (err error) {
c.AccessTiering, err = config.ParseBool(env.Get(EnvILMAccessTiering, kvs.GetWithDefault(accessTiering, DefaultKVS)))
if err != nil {
return err
}
if c.AccessPools, err = parseAccessPools(env.Get(EnvILMAccessPools, kvs.GetWithDefault(accessPools, DefaultKVS))); err != nil {
return err
}
// Enabling the feature without a usable topology is a configuration
// error worth surfacing at set time rather than silently doing nothing.
if c.AccessTiering && len(c.AccessPools) < 2 {
return ErrAccessPoolsTooFew
}
maxSize := env.Get(EnvILMAccessMaxSize, kvs.GetWithDefault(accessMaxSize, DefaultKVS))
if maxSize == "" || maxSize == "0" {
c.AccessMaxSize = 0
} else if c.AccessMaxSize, err = humanize.ParseBytes(maxSize); err != nil {
return err
}
if c.AccessPromoteWatermark, err = strconv.Atoi(env.Get(EnvILMAccessPromoteWatermark, kvs.GetWithDefault(accessPromoteWatermark, DefaultKVS))); err != nil {
return err
}
if c.AccessPromoteWatermark < 1 || c.AccessPromoteWatermark > 100 {
return ErrAccessWatermarkInvalid
}
if c.AccessBinWidth, err = time.ParseDuration(env.Get(EnvILMAccessBinWidth, kvs.GetWithDefault(accessBinWidth, DefaultKVS))); err != nil {
return err
}
if c.AccessBinWidth < time.Second {
return ErrAccessBinWidthInvalid
}
if c.AccessBins, err = strconv.Atoi(env.Get(EnvILMAccessBins, kvs.GetWithDefault(accessBins, DefaultKVS))); err != nil {
return err
}
if c.AccessBins < 2 || c.AccessBins > 64 {
return ErrAccessBinsInvalid
}
if c.AccessFlush, err = time.ParseDuration(env.Get(EnvILMAccessFlush, kvs.GetWithDefault(accessFlush, DefaultKVS))); err != nil {
return err
}
if c.AccessFlush < time.Second {
return ErrAccessFlushInvalid
}
if c.AccessMinResidency, err = time.ParseDuration(env.Get(EnvILMAccessMinResidency, kvs.GetWithDefault(accessMinResidency, DefaultKVS))); err != nil {
return err
}
if c.AccessMinResidency < 0 {
return ErrAccessResidencyInvalid
}
if c.AccessWorkers, err = strconv.Atoi(env.Get(EnvILMAccessWorkers, kvs.GetWithDefault(accessWorkers, DefaultKVS))); err != nil {
return err
}
if c.AccessWorkers < 1 {
return ErrAccessWorkersInvalid
}
if c.AccessMaxTracked, err = strconv.Atoi(env.Get(EnvILMAccessMaxTracked, kvs.GetWithDefault(accessMaxTracked, DefaultKVS))); err != nil {
return err
}
if c.AccessMaxTracked < 1 {
return ErrAccessTrackedInvalid
}
return nil
}
+109
View File
@@ -0,0 +1,109 @@
// Copyright (c) 2015-2026 MinIO, Inc.
package ilm
import (
"errors"
"reflect"
"testing"
"time"
"github.com/minio/minio/internal/config"
)
func clearAccessEnv(t *testing.T) {
t.Helper()
for _, name := range []string{
EnvILMAccessTiering, EnvILMAccessPools, EnvILMAccessMaxSize,
EnvILMAccessPromoteWatermark, EnvILMAccessBinWidth, EnvILMAccessBins,
EnvILMAccessFlush, EnvILMAccessMinResidency, EnvILMAccessWorkers,
EnvILMAccessMaxTracked,
} {
// The env helper treats an empty value as unset. t.Setenv restores the
// caller's exact value automatically when the test finishes.
t.Setenv(name, "")
}
}
func TestLookupAccessDefaults(t *testing.T) {
clearAccessEnv(t)
cfg, err := LookupConfig(DefaultKVS.Clone())
if err != nil {
t.Fatal(err)
}
if cfg.AccessTiering {
t.Fatal("access tiering must default off")
}
if cfg.AccessBinWidth != time.Minute || cfg.AccessBins != 12 || cfg.AccessFlush != time.Minute {
t.Fatalf("unexpected counter defaults: width=%s bins=%d flush=%s", cfg.AccessBinWidth, cfg.AccessBins, cfg.AccessFlush)
}
if cfg.AccessWorkers != 10 || cfg.AccessMaxTracked != 1000000 {
t.Fatalf("unexpected worker/map defaults: %d/%d", cfg.AccessWorkers, cfg.AccessMaxTracked)
}
}
func TestLookupAccessEnabled(t *testing.T) {
clearAccessEnv(t)
kvs := DefaultKVS.Clone()
kvs.Set(accessTiering, config.EnableOn)
kvs.Set(accessPools, "2, 0, 1")
kvs.Set(accessMaxSize, "2GiB")
kvs.Set(accessPromoteWatermark, "90")
kvs.Set(accessBinWidth, "30s")
kvs.Set(accessBins, "20")
kvs.Set(accessFlush, "15s")
kvs.Set(accessMinResidency, "2h")
kvs.Set(accessWorkers, "7")
kvs.Set(accessMaxTracked, "1234")
cfg, err := LookupConfig(kvs)
if err != nil {
t.Fatal(err)
}
if !cfg.AccessTiering || !reflect.DeepEqual(cfg.AccessPools, []int{2, 0, 1}) {
t.Fatalf("unexpected topology: enabled=%v pools=%v", cfg.AccessTiering, cfg.AccessPools)
}
if cfg.AccessMaxSize != 2<<30 || cfg.HistoryWindow() != 10*time.Minute {
t.Fatalf("unexpected size/history: %d/%s", cfg.AccessMaxSize, cfg.HistoryWindow())
}
if hot, ok := cfg.HotPool(); !ok || hot != 2 {
t.Fatalf("HotPool = %d/%v", hot, ok)
}
if cold, ok := cfg.ColdPool(); !ok || cold != 1 {
t.Fatalf("ColdPool = %d/%v", cold, ok)
}
}
func TestLookupAccessRejectsInvalidValues(t *testing.T) {
clearAccessEnv(t)
tests := []struct {
key, value string
want error
}{
{accessPools, "0,0", ErrAccessPoolsInvalid},
{accessPromoteWatermark, "0", ErrAccessWatermarkInvalid},
{accessBinWidth, "500ms", ErrAccessBinWidthInvalid},
{accessBins, "1", ErrAccessBinsInvalid},
{accessFlush, "0s", ErrAccessFlushInvalid},
{accessMinResidency, "-1s", ErrAccessResidencyInvalid},
{accessWorkers, "0", ErrAccessWorkersInvalid},
{accessMaxTracked, "0", ErrAccessTrackedInvalid},
}
for _, tc := range tests {
t.Run(tc.key, func(t *testing.T) {
kvs := DefaultKVS.Clone()
kvs.Set(tc.key, tc.value)
_, err := LookupConfig(kvs)
if !errors.Is(err, tc.want) {
t.Fatalf("error = %v, want %v", err, tc.want)
}
})
}
kvs := DefaultKVS.Clone()
kvs.Set(accessTiering, config.EnableOn)
kvs.Set(accessPools, "0")
if _, err := LookupConfig(kvs); !errors.Is(err, ErrAccessPoolsTooFew) {
t.Fatalf("error = %v, want %v", err, ErrAccessPoolsTooFew)
}
}