use the new errgroup API whereever applicable (#11466)

start using the new errgroup concurrency control
API introduced in #11457
This commit is contained in:
Harshavardhana
2021-02-09 12:08:25 -08:00
committed by GitHub
parent 84d400487f
commit fe3c39b583
2 changed files with 22 additions and 26 deletions
+14 -20
View File
@@ -31,27 +31,21 @@ import (
)
func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
inParallel := func(objects []ObjectInfo) {
g := errgroup.WithNErrs(len(objects))
for index := range objects {
index := index
g.Go(func() error {
objects[index].ETag = objects[index].GetActualETag(nil)
objects[index].Size, _ = objects[index].GetActualSize()
return nil
}, index)
}
g.Wait()
}
const maxConcurrent = 500
for {
if len(objects) < maxConcurrent {
inParallel(objects)
return
}
inParallel(objects[:maxConcurrent])
objects = objects[maxConcurrent:]
g := errgroup.WithNErrs(len(objects)).WithConcurrency(500)
_, cancel := g.WithCancelOnError(ctx)
defer cancel()
for index := range objects {
index := index
g.Go(func() error {
size, err := objects[index].GetActualSize()
if err == nil {
objects[index].Size = size
}
objects[index].ETag = objects[index].GetActualETag(nil)
return nil
}, index)
}
g.WaitErr()
}
// Validate all the ListObjects query arguments, returns an APIErrorCode