diff --git a/cmd/erasure-healing-native-audit_test.go b/cmd/erasure-healing-native-audit_test.go new file mode 100644 index 000000000..bb9b02a13 --- /dev/null +++ b/cmd/erasure-healing-native-audit_test.go @@ -0,0 +1,244 @@ +// Copyright (c) 2026 Jasper Bernhardt +// +// This file is part of Silo Object Storage. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/minio/madmin-go/v3" +) + +// Unimplemented operations panic: the dry-run tests only allow reads and checks. +type nativeAuditDisk struct { + StorageAPI + info FileInfo + readErr error + verify func() (*CheckPartsResp, error) + deletes atomic.Int32 +} + +func (d *nativeAuditDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) { + return d.info, d.readErr +} + +func (d *nativeAuditDisk) VerifyFile(context.Context, string, string, FileInfo) (*CheckPartsResp, error) { + return d.verify() +} + +func (d *nativeAuditDisk) DeleteVersion(context.Context, string, string, FileInfo, bool, DeleteOptions) error { + d.deletes.Add(1) + return nil +} + +func nativeAuditMetadata() []FileInfo { + meta := make([]FileInfo, 4) + for i := range meta { + meta[i] = newFileInfo("object", 2, 2) + meta[i].Volume, meta[i].Name = "bucket", "object" + meta[i].DataDir = "data" + meta[i].ModTime = time.Unix(123, 0).UTC() + meta[i].Size = 1024 + meta[i].Erasure.Distribution = []int{1, 2, 3, 4} + meta[i].Erasure.Index = i + 1 + meta[i].Parts = []ObjectPartInfo{{Number: 1, Size: 512}, {Number: 2, Size: 512}} + } + return meta +} + +func nativeAuditEndpoints() []Endpoint { + endpoints := make([]Endpoint, 4) + for i := range endpoints { + endpoints[i].URL = &url.URL{Path: fmt.Sprintf("/synthetic/disk%d", i)} + } + return endpoints +} + +func TestNativeAuditDryRunPreservesFaults(t *testing.T) { + for _, fault := range []string{"metadata-quorum", "corrupt-metadata", "missing-parts", "corrupt-parts", "inline-corrupt", "delete-marker-quorum"} { + t.Run(fault, func(t *testing.T) { + meta := nativeAuditMetadata() + disks := make([]StorageAPI, len(meta)) + for i := range disks { + disk := &nativeAuditDisk{info: meta[i], verify: func() (*CheckPartsResp, error) { + return &CheckPartsResp{Results: []int{checkPartSuccess, checkPartSuccess}}, nil + }} + if i < 3 { + switch fault { + case "metadata-quorum", "delete-marker-quorum": + disk.readErr = errFileVersionNotFound + case "corrupt-metadata": + disk.readErr = errFileCorrupt + case "missing-parts": + disk.verify = func() (*CheckPartsResp, error) { return nil, errFileNotFound } + case "corrupt-parts": + disk.verify = func() (*CheckPartsResp, error) { return nil, errFileCorrupt } + } + } + if fault == "delete-marker-quorum" { + disk.info.Deleted = true + disk.info.Size = 0 + } + if fault == "inline-corrupt" { + disk.info.Data = []byte("corrupt") + disk.info.Parts = disk.info.Parts[:1] + disk.info.SetInlineData() + disk.info.Erasure.Checksums = []ChecksumInfo{{PartNumber: 1, Algorithm: SHA256, Hash: make([]byte, sha256.Size)}} + } + disks[i] = disk + } + er := erasureObjects{ + setDriveCount: 4, defaultParityCount: 2, + getDisks: func() []StorageAPI { return disks }, + getEndpoints: nativeAuditEndpoints, + } + result, err := er.HealObject(t.Context(), "bucket", "object", "version", madmin.HealOpts{DryRun: true, NoLock: true, Remove: true, ScanMode: madmin.HealDeepScan}) + for _, disk := range disks { + if got := disk.(*nativeAuditDisk).deletes.Load(); got != 0 { + t.Fatalf("dry-run deleted data %d times", got) + } + } + quorumFault := fault == "metadata-quorum" || fault == "corrupt-metadata" || fault == "delete-marker-quorum" + if quorumFault && err == nil { + t.Fatal("metadata quorum failure reported success") + } + if !quorumFault && err != nil { + t.Fatal(err) + } + if len(result.Before.Drives) != len(disks) || !reflect.DeepEqual(result.Before, result.After) { + t.Fatalf("dry-run states must preserve observed damage: %+v", result) + } + switch fault { + case "metadata-quorum", "delete-marker-quorum": + if !isErrVersionNotFound(err) { + t.Fatalf("dry-run must return the reduced metadata error, got %v", err) + } + case "corrupt-metadata": + if !errors.Is(err, errFileCorrupt) { + t.Fatalf("dry-run must return the reduced metadata error, got %v", err) + } + } + unhealthy := 0 + for _, drive := range result.Before.Drives { + if drive.State != madmin.DriveStateOk { + unhealthy++ + } + } + if unhealthy < 3 { + t.Fatalf("faults were not reported: %+v", result.Before) + } + }) + } +} + +// This control proves the missing-metadata fixture reaches dangling deletion. +func TestNativeAuditNonDryRunStillDeletesDangling(t *testing.T) { + meta := nativeAuditMetadata() + disks := make([]StorageAPI, len(meta)) + for i := range disks { + disk := &nativeAuditDisk{info: meta[i]} + if i < 3 { + disk.readErr = errFileVersionNotFound + } + disks[i] = disk + } + er := erasureObjects{ + setDriveCount: 4, defaultParityCount: 2, + getDisks: func() []StorageAPI { return disks }, + getEndpoints: nativeAuditEndpoints, + } + _, err := er.HealObject(t.Context(), "bucket", "object", "version", madmin.HealOpts{NoLock: true, Remove: true, ScanMode: madmin.HealDeepScan}) + if err == nil { + t.Fatal("dangling deletion must report the object removed") + } + for _, disk := range disks { + if got := disk.(*nativeAuditDisk).deletes.Load(); got != 1 { + t.Fatalf("control did not reach dangling deletion: %d", got) + } + } +} + +func TestNativeAuditDryRunPreservesDanglingFiles(t *testing.T) { + obj, roots, err := prepareErasure(t.Context(), 4) + if err != nil { + t.Fatal(err) + } + setObjectLayer(obj) + defer removeRoots(roots) + defer obj.Shutdown(t.Context()) + const bucket, object = "native-audit", "dangling-object" + if err = obj.MakeBucket(t.Context(), bucket, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + data := bytes.Repeat([]byte("x"), 1024*1024) + if _, err = obj.PutObject(t.Context(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}); err != nil { + t.Fatal(err) + } + for _, root := range roots[:3] { + if err = os.Remove(filepath.Join(root, bucket, object, xlStorageFormatFile)); err != nil { + t.Fatal(err) + } + } + snapshot := func() map[string]string { + files := make(map[string]string) + for _, root := range roots { + err := filepath.Walk(filepath.Join(root, bucket, object), func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + files[path] = fmt.Sprintf("%d/%d/%x", info.Size(), info.ModTime().UnixNano(), sha256.Sum256(data)) + return nil + }) + if err != nil { + t.Fatal(err) + } + } + return files + } + before := snapshot() + if len(before) < 2 { + t.Fatal("fixture has no surviving metadata and payload files") + } + result, err := obj.HealObject(t.Context(), bucket, object, "", madmin.HealOpts{DryRun: true, Remove: true, ScanMode: madmin.HealDeepScan}) + if err == nil { + t.Fatal("missing metadata quorum reported success") + } + if !reflect.DeepEqual(result.Before, result.After) { + t.Fatalf("dry-run reported changes: %+v", result) + } + if after := snapshot(); !reflect.DeepEqual(before, after) { + t.Fatalf("dry-run changed surviving object files: before=%v after=%v", before, after) + } +} diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go index 8fb371270..17e1e8cd5 100644 --- a/cmd/erasure-healing.go +++ b/cmd/erasure-healing.go @@ -343,6 +343,11 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) if err != nil { + if dryRun { + // A metadata quorum failure must not delete dangling data in dry-run. + return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, + errs, bucket, object, versionID), err + } m, derr := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, nil, ObjectOptions{ VersionID: versionID, })