retry disk replacement healing if listing fails (#13689)

listing can fail and it is allowed to be retried,
instead of returning right away return an error at
the end - heal the rest of the buckets and objects,
and when we are retrying skip the buckets that
are already marked done by using the tracked buckets.

fixes #12972
This commit is contained in:
Harshavardhana
2021-11-19 08:46:47 -08:00
committed by GitHub
parent 81d19156e9
commit 17fd71164c
2 changed files with 37 additions and 28 deletions
+6 -6
View File
@@ -18,12 +18,12 @@
package cmd
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"sort"
"strings"
"sync"
@@ -412,7 +412,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
// So someone changed the drives underneath, healing tracker missing.
tracker, err := loadHealingTracker(ctx, disk)
if err != nil {
logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool",
disk, humanize.Ordinal(i+1))
tracker = newHealingTracker(disk)
}
@@ -434,16 +435,15 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
return
}
err = z.serverPools[i].sets[setIndex].healErasureSet(ctx, buckets, tracker)
err = z.serverPools[i].sets[setIndex].healErasureSet(ctx, tracker.QueuedBuckets, tracker)
if err != nil {
logger.LogIf(ctx, err)
continue
}
logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
var buf bytes.Buffer
tracker.printTo(&buf)
logger.Info("Summary:\n%s", buf.String())
logger.Info("Summary:\n")
tracker.printTo(os.Stdout)
logger.LogIf(ctx, tracker.delete(ctx))
// Only upon success pop the healed disk.