diff --git a/cmd/erasure-multipart-cleanup_test.go b/cmd/erasure-multipart-cleanup_test.go new file mode 100644 index 000000000..3181a4b1e --- /dev/null +++ b/cmd/erasure-multipart-cleanup_test.go @@ -0,0 +1,177 @@ +// Copyright (c) 2026 clown15 +// +// This file is part of Silo Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "fmt" + "net/url" + "os" + "path/filepath" + "testing" + "time" +) + +func TestExtractModTimeFromUploadIDDir(t *testing.T) { + now := time.Unix(1792056747, 123456789).UTC() + nano := now.UnixNano() + uuid := "d16ad42a-6fe5-4a3b-8092-69983142238f" + + testCases := []struct { + name string + dirName string + expectedMod time.Time + wantZero bool + }{ + { + name: "Modern Plaintext Format (x)", + dirName: fmt.Sprintf("%sx%d", uuid, nano), + expectedMod: time.Unix(0, nano), + wantZero: false, + }, + { + // This valid UUID decodes to bytes containing 'x'. Checking + // for that byte before parsing plaintext loses the timestamp. + name: "Plaintext Also Decodes To Base64 Containing x", + dirName: fmt.Sprintf("4eacd477-e733-4851-b821-6fd42785ff87x%d", nano), + expectedMod: now, + }, + { + name: "Legacy Format Without Timestamp ()", + dirName: uuid, + expectedMod: time.Time{}, + wantZero: true, + }, + { + name: "Base64 Encoded Format", + dirName: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%sx%d", uuid, nano))), + expectedMod: time.Unix(0, nano), + wantZero: false, + }, + { + name: "Encoded Format With Deployment Prefix", + dirName: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%sx%d", uuid, uuid, nano))), + expectedMod: now, + }, + { + name: "Malformed Suffix (xinvalid)", + dirName: fmt.Sprintf("%sxnotanumber", uuid), + expectedMod: time.Time{}, + wantZero: true, + }, + { + name: "Timestamp Overflow", + dirName: uuid + "x9223372036854775808", + wantZero: true, + }, + { + name: "Malformed Plaintext UUID", + dirName: fmt.Sprintf("not-a-uuidx%d", nano), + wantZero: true, + }, + { + name: "Empty Directory Name", + dirName: "", + expectedMod: time.Time{}, + wantZero: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got := extractModTimeFromUploadIDDir(tc.dirName) + if tc.wantZero { + if !got.IsZero() { + t.Fatalf("expected zero time for %q, got %v", tc.dirName, got) + } + return + } + if !got.Equal(tc.expectedMod) { + t.Fatalf("extractModTimeFromUploadIDDir(%q) = %v; want %v", tc.dirName, got, tc.expectedMod) + } + }) + } +} + +type multipartCleanupDisk struct { + StorageAPI + root string + modTime time.Time + readErr error + reads int +} + +func (d *multipartCleanupDisk) Endpoint() Endpoint { + return Endpoint{URL: &url.URL{Path: d.root}} +} + +func (d *multipartCleanupDisk) ReadVersion(context.Context, string, string, string, string, ReadOptions) (FileInfo, error) { + d.reads++ + return FileInfo{ModTime: d.modTime}, d.readErr +} + +func (d *multipartCleanupDisk) StatVol(context.Context, string) (VolInfo, error) { + return VolInfo{Created: time.Now()}, nil +} + +func TestCleanupStaleUploadsOnDisk(t *testing.T) { + const uuid = "d16ad42a-6fe5-4a3b-8092-69983142238f" + expiry := globalAPIConfig.getStaleUploadsExpiry() + for _, tc := range []struct { + name, prefix string + age time.Duration + legacy bool + readErr error + wantReads int + wantDeleted bool + }{ + {name: "recent", prefix: uuid, age: expiry / 2, readErr: errFileNotFound}, + {name: "plaintext_base64_collision", prefix: "4eacd477-e733-4851-b821-6fd42785ff87", age: expiry / 2, readErr: errFileNotFound}, + {name: "expired", prefix: uuid, age: 2 * expiry, readErr: errFileNotFound, wantDeleted: true}, + {name: "legacy_recent", prefix: uuid, age: expiry / 2, legacy: true, wantReads: 1}, + {name: "legacy_expired", prefix: uuid, age: 2 * expiry, legacy: true, wantReads: 1, wantDeleted: true}, + {name: "legacy_metadata_missing", prefix: uuid, age: 2 * expiry, legacy: true, readErr: errFileNotFound, wantReads: 1}, + {name: "malformed_metadata_missing", prefix: "not-a-uuid", age: 2 * expiry, readErr: errFileNotFound, wantReads: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + modTime := time.Now().Add(-tc.age) + id := tc.prefix + if !tc.legacy { + id = fmt.Sprintf("%sx%d", tc.prefix, modTime.UnixNano()) + } + disk := &multipartCleanupDisk{root: t.TempDir(), modTime: modTime, readErr: tc.readErr} + uploadDir := filepath.Join(disk.root, minioMetaMultipartBucket, "sha", id) + if err := os.MkdirAll(uploadDir, 0o755); err != nil { + t.Fatal(err) + } + erasureObjects{}.cleanupStaleUploadsOnDisk(t.Context(), disk) + if disk.reads != tc.wantReads { + t.Errorf("ReadVersion calls = %d, want %d", disk.reads, tc.wantReads) + } + _, err := os.Stat(uploadDir) + if tc.wantDeleted { + if !os.IsNotExist(err) { + t.Errorf("expired upload must be removed: %v", err) + } + } else if err != nil { + t.Errorf("upload must be preserved: %v", err) + } + }) + } +} diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go index 7da35c014..5e57cbd17 100644 --- a/cmd/erasure-multipart.go +++ b/cmd/erasure-multipart.go @@ -174,6 +174,24 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) { wg.Wait() } +// extractModTimeFromUploadIDDir extracts creation time from the upload ID directory on disk. +// On-disk directory names are of the form x. +// Parse the native format first: plaintext UUIDs can also be valid Base64. +// Preserve the previous encoded-format parser for compatibility. +func extractModTimeFromUploadIDDir(uploadIDDir string) time.Time { + if modTime, ok := multipartUploadTime(uploadIDDir); ok { + return modTime + } + if b64, err := base64.RawURLEncoding.DecodeString(uploadIDDir); err == nil { + if split := strings.Split(string(b64), "x"); len(split) == 2 { + if t, err := strconv.ParseInt(split[1], 10, 64); err == nil { + return time.Unix(0, t) + } + } + } + return time.Time{} +} + // Remove the old multipart uploads on the given disk. func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI) { drivePath := disk.Endpoint().Path @@ -181,16 +199,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error { readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error { uploadIDPath := pathJoin(shaDir, uploadIDDir) - var modTime time.Time - // Upload IDs are of the form base64_url(x), we can extract the time from the UUID. - if b64, err := base64.RawURLEncoding.DecodeString(uploadIDDir); err == nil { - if split := strings.Split(string(b64), "x"); len(split) == 2 { - t, err := strconv.ParseInt(split[1], 10, 64) - if err == nil { - modTime = time.Unix(0, t) - } - } - } + modTime := extractModTimeFromUploadIDDir(uploadIDDir) // Fallback for older uploads without time in the ID. if modTime.IsZero() { wait := deleteMultipartCleanupSleeper.Timer(ctx)