astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-08-10 16:23:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Allow compaction at bucket top level. (#15266)

Browse Source 
If more than 1M folders (objects or prefixes) are found at the top level in a bucket allow it to be compacted.

While very suboptimal structure we should limit memory usage at some point.
This commit is contained in:
Klaus Post
2022-07-11 07:59:03 -07:00
committed by GitHub
parent 913e977c8d
commit 37a6b2da67
1 changed files with 12 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/data-scanner.go
+6 -3
View File
@@ -53,6 +53,7 @@ const (
dataScannerCompactLeastObject = 500 // Compact when there is less than this many objects in a branch.
dataScannerCompactAtChildren = 10000 // Compact when there are this many children in a branch.
dataScannerCompactAtFolders = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder.
dataScannerForceCompactAtFolders = 1_000_000 // Compact when this many subfolders in a single folder (even top level).
dataScannerStartDelay = 1 * time.Minute // Time to wait on startup and between cycles.
healDeleteDangling = true
@@ -566,9 +567,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
}
// If we have many subfolders, compact ourself.
if !into.Compacted &&
f.newCache.Info.Name != folder.name &&
len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders {
shouldCompact := f.newCache.Info.Name != folder.name &&
len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
if !into.Compacted && shouldCompact {
into.Compacted = true
newFolders = append(newFolders, existingFolders...)
existingFolders = nil