astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-25 06:56:18 +03:00
Code Issues Packages Projects Releases Wiki Activity

Make WalkDir return errors (#19677)

Browse Source 
If used, 'opts.Marker` will cause many missed entries since results are returned 
unsorted, and pools are serialized.

Switch to fully concurrent listing and merging across pools to return sorted entries.
This commit is contained in:
Klaus Post
2024-05-06 13:27:52 -07:00
committed by GitHub
parent 9a9a49aa84
commit 847ee5ac45
9 changed files with 169 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/erasure-server-pool.go
+22 -9
View File
@@ -2045,13 +2045,13 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
// Walk a bucket, optionally prefix recursively, until we have returned
// all the contents of the provided bucket+prefix.
// TODO: Note that most errors will result in a truncated listing.
func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, opts WalkOptions) error {
func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error {
if err := checkListObjsArgs(ctx, bucket, prefix, ""); err != nil {
xioutil.SafeClose(results)
return err
}
ctx, cancel := context.WithCancel(ctx)
parentCtx := ctx
ctx, cancelCause := context.WithCancelCause(ctx)
var entries []chan metaCacheEntry
for poolIdx, erasureSet := range z.serverPools {
@@ -2062,8 +2062,9 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
disks, infos, _ := set.getOnlineDisksWithHealingAndInfo(true)
if len(disks) == 0 {
xioutil.SafeClose(results)
cancel()
return fmt.Errorf("Walk: no online disks found in pool %d, set %d", setIdx, poolIdx)
err := fmt.Errorf("Walk: no online disks found in pool %d, set %d", setIdx, poolIdx)
cancelCause(err)
return err
}
go func() {
defer xioutil.SafeClose(listOut)
@@ -2150,8 +2151,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
}
if err := listPathRaw(ctx, lopts); err != nil {
storageLogIf(ctx, fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
cancel()
cancelCause(fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts))
return
}
}()
@@ -2162,13 +2162,24 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
merged := make(chan metaCacheEntry, 100)
vcfg, _ := globalBucketVersioningSys.Get(bucket)
go func() {
defer cancel()
defer cancelCause(nil)
defer xioutil.SafeClose(results)
sentErr := false
sendErr := func(err error) {
if !sentErr {
select {
case results <- itemOrErr[ObjectInfo]{Err: err}:
sentErr = true
case <-parentCtx.Done():
}
}
}
send := func(oi ObjectInfo) bool {
select {
case results <- oi:
case results <- itemOrErr[ObjectInfo]{Item: oi}:
return true
case <-ctx.Done():
sendErr(context.Cause(ctx))
return false
}
}
@@ -2176,6 +2187,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
if opts.LatestOnly {
fi, err := entry.fileInfo(bucket)
if err != nil {
sendErr(err)
return
}
if opts.Filter != nil {
@@ -2193,6 +2205,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
}
fivs, err := entry.fileInfoVersions(bucket)
if err != nil {
sendErr(err)
return
}