revert: remove access-frequency ILM tiering (#60)

Reverse the first-parent diff of a3df317ae0, including the feature branch compatibility and mover follow-up fixes. Retain the independent multi-pool correctness fixes from #178 and migrate their shared test fixture away from access-tier code. Tolerate retired ILM keys and XML, read old v9 statistics while writing v8, and document migration without moving objects or rewriting their metadata. Include regression coverage using a historical scanner/writer v9 fixture. Signed-off-by: Feng Ruohang <rh@vonng.com>
2026-09-15 15:04:04 +03:00 · 2026-09-15 06:51:14 +08:00
parent 89637554d6
commit 9b76a21675
44 changed files with 581 additions and 5670 deletions
@@ -0,0 +1,27 @@
+# Data-usage v9 retirement fixture
+
+Generated using unmodified `scanDataFolder` and `dataUsageCache.serializeTo` from
+Server commit `89637554d60c27cfc51d2281d0a4fe15e415f06d` (Go 1.27.1, darwin/arm64).
+The scanner visits three real local files; its size callback supplies synthetic
+version/delete-marker/remote-tier summaries, following `TestDataUsageCacheSerialize`.
+It is a scanner/cache compatibility fixture, not a distributed object-store test.
+
+The old scanner counts 74,962 bytes, 3 objects, 5 versions and 2 delete markers.
+Its nonzero retired `hts` totals 9,426 bytes. Remote tier `COLD` contains 65,536
+bytes, one version and one object. The cache includes nested children, both
+histograms, a fixed timestamp and scanner cycle 42. The JSON is the old scanner's
+complete expected cache (including `HotTierSize`, which the new reader ignores).
+
+Binary SHA-256: `4c9c7e94cea7758fe9b498b19f639188764b2787582783e6cc950f2c44d52a8b`.
+
+To regenerate, create a detached worktree at that exact source commit, copy
+`generate.go.txt` to `cmd/retirement-fixture_test.go`, and run:
+
+```sh
+SILO_RETIRE_FIXTURE_DIR=/absolute/output/directory go test ./cmd -run '^TestGenerateAccessRetirementV9Fixture$' -count=1 -v
+```
+
+The historical production writer adds the version byte, compresses with zstd
+and encodes msgp, including the nonzero `hts` field. Do not regenerate using the
+retired implementation or by changing the header of a v8 payload. Map order may
+change serialized bytes across regeneration; compare the decoded full cache.
@@ -0,0 +1,118 @@
+{
+  "Info": {
+    "Name": "/",
+    "NextCycle": 42,
+    "LastUpdate": "2026-09-15T00:00:00Z",
+    "SkipHealing": true
+  },
+  "Cache": {
+    "/": {
+      "Children": {
+        "v9-bucket": {}
+      },
+      "Size": 0,
+      "HotTierSize": 0,
+      "Objects": 0,
+      "Versions": 0,
+      "DeleteMarkers": 0,
+      "ObjSizes": [
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "ObjVersions": [
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "AllTierStats": null,
+      "Compacted": false
+    },
+    "v9-bucket": {
+      "Children": {
+        "v9-bucket/nested": {}
+      },
+      "Size": 1234,
+      "HotTierSize": 1234,
+      "Objects": 1,
+      "Versions": 1,
+      "DeleteMarkers": 0,
+      "ObjSizes": [
+        0,
+        1,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "ObjVersions": [
+        0,
+        1,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "AllTierStats": null,
+      "Compacted": false
+    },
+    "v9-bucket/nested": {
+      "Children": null,
+      "Size": 73728,
+      "HotTierSize": 8192,
+      "Objects": 2,
+      "Versions": 4,
+      "DeleteMarkers": 2,
+      "ObjSizes": [
+        0,
+        1,
+        1,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0,
+        0
+      ],
+      "ObjVersions": [
+        0,
+        1,
+        1,
+        0,
+        0,
+        0,
+        0
+      ],
+      "AllTierStats": {
+        "Tiers": {
+          "COLD": {
+            "TotalSize": 65536,
+            "NumVersions": 1,
+            "NumObjects": 1
+          }
+        }
+      },
+      "Compacted": true
+    }
+  }
+}
@@ -0,0 +1,82 @@
+// Copyright (c) 2026 Feng Ruohang
+//
+// This file is part of Silo Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/minio/minio/internal/cachevalue"
+)
+
+// Run only on 89637554d, before access-tiering is removed. This uses the real
+// folder scanner and v9 serializer with synthetic file-size/tier summaries,
+// following TestDataUsageCacheSerialize; it does not relabel a v8 payload.
+func TestGenerateAccessRetirementV9Fixture(t *testing.T) {
+ if dataUsageCacheVerCurrent != 9 { t.Fatal("requires the historical v9 writer") }
+ base := t.TempDir()
+ const bucket = "v9-bucket"
+ createUsageTestFiles(t, base, bucket, []usageTestFile{
+  {name: "root", size: 1234},
+  {name: "nested/versions", size: 8192},
+  {name: "nested/remote", size: 65536},
+ })
+ getSize := func(item scannerItem) (s sizeSummary, err error) {
+  if item.Typ&os.ModeDir != 0 { return s, nil }
+  info, err := os.Stat(item.Path)
+  if err != nil { return s, err }
+  s.totalSize, s.hotTierSize, s.versions = info.Size(), info.Size(), 1
+  if filepath.Base(item.Path) == "versions" { s.versions, s.deleteMarkers = 3, 2 }
+  if filepath.Base(item.Path) == "remote" {
+   s.hotTierSize = 0
+   s.tiers = map[string]tierStats{"COLD": {TotalSize: uint64(info.Size()), NumVersions: 1, NumObjects: 1}}
+  }
+  return s, nil
+ }
+ xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()}
+ xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(context.Context) (DiskInfo,error) {
+  return DiskInfo{Total: 1<<40, Free: 1<<40}, nil
+ })
+ cache, err := scanDataFolder(t.Context(), nil, &xls, dataUsageCache{Info:dataUsageCacheInfo{Name:bucket, SkipHealing:true}}, getSize, 0, func()bool{return false})
+ if err != nil { t.Fatal(err) }
+ root := *cache.find(bucket)
+ cache.replace(dataUsageRoot, "", dataUsageEntry{})
+ cache.replace(bucket, dataUsageRoot, root)
+ cache.Info.Name = dataUsageRoot
+ cache.Info.LastUpdate = time.Date(2026, 9, 15, 0, 0, 0, 0, time.UTC)
+ cache.Info.NextCycle = 42
+ flat := cache.flatten(*cache.root())
+ if flat.Size != 74962 || flat.Objects != 3 || flat.Versions != 5 || flat.DeleteMarkers != 2 || flat.HotTierSize != 9426 {
+  t.Fatalf("unexpected scanner fixture: %+v", flat)
+ }
+ var buf bytes.Buffer
+ if err := cache.serializeTo(&buf); err != nil { t.Fatal(err) }
+ dir := os.Getenv("SILO_RETIRE_FIXTURE_DIR")
+ if dir == "" { t.Fatal("SILO_RETIRE_FIXTURE_DIR required") }
+ if err := os.MkdirAll(dir, 0755); err != nil { t.Fatal(err) }
+ if err := os.WriteFile(filepath.Join(dir,"data-usage-v9.bin"),buf.Bytes(),0644);err != nil{t.Fatal(err)}
+ expected, err := json.MarshalIndent(cache,"","  ")
+ if err != nil { t.Fatal(err) }
+ if err := os.WriteFile(filepath.Join(dir,"data-usage-v9.json"),append(expected,'\n'),0644);err != nil{t.Fatal(err)}
+ t.Logf("old scanner/v9 writer: bytes=%d size=%d objects=%d versions=%d markers=%d hts=%d",buf.Len(),flat.Size,flat.Objects,flat.Versions,flat.DeleteMarkers,flat.HotTierSize)
+}