astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-23 14:10:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

replication: improve MRF healing. (#15556)

Browse Source 
This PR improves the replication failure healing by persisting
most recent failures to disk and re-queuing them until the replication
is successful.

While this does not eliminate the need for healing during a full scan, 
queuing MRF vastly improves the ETA to keeping replicated buckets 
in sync as it does not wait for the scanner visit to detect unreplicated 
object versions.
This commit is contained in:
Poorna
2022-08-22 16:53:06 -07:00
committed by GitHub
parent 471467d310
commit 4155c5b695
4 changed files with 886 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/bucket-replication.go
+224 -28
View File
@@ -403,6 +403,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
if err != nil {
globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, rcfg.RoleArn))
sendEvent(eventArgs{
BucketName: bucket,
@@ -477,6 +478,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
eventName := event.ObjectReplicationComplete
if replicationStatus == replication.Failed {
eventName = event.ObjectReplicationFailed
globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
}
drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
if replicationStatus != prevStatus {
@@ -910,6 +912,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
Object: objInfo,
Host: "Internal: [Replication]",
})
globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, cfg.RoleArn))
return
}
@@ -992,12 +995,12 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
// re-queue failures once more - keep a retry count to avoid flooding the queue if
// the target site is down. Leave it to scanner to catch up instead.
if rinfos.ReplicationStatus() != replication.Completed && ri.RetryCount < 1 {
if rinfos.ReplicationStatus() != replication.Completed {
ri.OpType = replication.HealReplicationType
ri.EventType = ReplicateMRF
ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal()
ri.RetryCount++
globalReplicationPool.queueReplicaFailedTask(ri)
globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
}
}
@@ -1304,6 +1307,19 @@ type DeletedObjectReplicationInfo struct {
TargetArn string
}
// ToMRFEntry returns the relevant info needed by MRF
func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry {
versionID := di.DeleteMarkerVersionID
if versionID == "" {
versionID = di.VersionID
}
return MRFReplicateEntry{
Bucket: di.Bucket,
Object: di.ObjectName,
versionID: versionID,
}
}
// Replication specific APIName
const (
ReplicateObjectAPI = "ReplicateObject"
@@ -1348,13 +1364,16 @@ type ReplicationPool struct {
mrfReplicaCh chan ReplicateObjectInfo
existingReplicaCh chan ReplicateObjectInfo
existingReplicaDeleteCh chan DeletedObjectReplicationInfo
workerSize int
mrfWorkerSize int
resyncState replicationResyncState
workerWg sync.WaitGroup
mrfWorkerWg sync.WaitGroup
once sync.Once
mu sync.Mutex
mrfSaveCh chan MRFReplicateEntry
workerSize int
mrfWorkerSize int
resyncState replicationResyncState
workerWg sync.WaitGroup
mrfWorkerWg sync.WaitGroup
once sync.Once
mu sync.Mutex
mrfMutex sync.Mutex
}
// NewReplicationPool creates a pool of replication workers of specified size
@@ -1368,6 +1387,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
existingReplicaCh: make(chan ReplicateObjectInfo, 100000),
existingReplicaDeleteCh: make(chan DeletedObjectReplicationInfo, 100000),
resyncState: replicationResyncState{statusMap: make(map[string]BucketReplicationResyncStatus)},
mrfSaveCh: make(chan MRFReplicateEntry, 100000),
ctx: ctx,
objLayer: o,
}
@@ -1376,6 +1396,8 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
pool.ResizeFailedWorkers(opts.FailedWorkers)
go pool.AddExistingObjectReplicateWorker()
go pool.updateResyncStatus(ctx, o)
go pool.processMRF()
go pool.persistMRF()
return pool
}
@@ -1481,33 +1503,17 @@ func (p *ReplicationPool) suggestedWorkers(failQueue bool) int {
return int(float64(p.workerSize) * ReplicationWorkerMultiplier)
}
func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
if p == nil {
return
}
select {
case <-GlobalContext.Done():
p.once.Do(func() {
close(p.replicaCh)
close(p.mrfReplicaCh)
close(p.existingReplicaCh)
})
case p.mrfReplicaCh <- ri:
default:
logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up retrying failed replication - we recommend increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), string(replicationSubsystem))
}
}
func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
if p == nil {
return
}
var ch chan ReplicateObjectInfo
var ch, healCh chan ReplicateObjectInfo
switch ri.OpType {
case replication.ExistingObjectReplicationType:
ch = p.existingReplicaCh
case replication.HealReplicationType:
fallthrough
ch = p.mrfReplicaCh
healCh = p.replicaCh
default:
ch = p.replicaCh
}
@@ -1518,6 +1524,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
close(p.mrfReplicaCh)
close(p.existingReplicaCh)
})
case healCh <- ri:
case ch <- ri:
default:
logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Unable to keep up with incoming traffic - we recommend increasing number of replicate object workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), string(replicationSubsystem))
@@ -2475,3 +2482,192 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
}
return
}
const mrfTimeInterval = 5 * time.Minute
func (p *ReplicationPool) persistMRF() {
var mu sync.Mutex
entries := make(map[string]MRFReplicateEntry)
mTimer := time.NewTimer(mrfTimeInterval)
defer mTimer.Stop()
saveMRFToDisk := func(drain bool) {
mu.Lock()
defer mu.Unlock()
if len(entries) == 0 {
return
}
cctx := p.ctx
if drain {
cctx = context.Background()
// drain all mrf entries and save to disk
for e := range p.mrfSaveCh {
entries[e.versionID] = e
}
}
if err := p.saveMRFEntries(cctx, entries); err != nil {
logger.LogOnceIf(p.ctx, fmt.Errorf("Unable to persist replication failures to disk:%w", err), string(replicationSubsystem))
}
entries = make(map[string]MRFReplicateEntry)
return
}
for {
select {
case <-mTimer.C:
saveMRFToDisk(false)
mTimer.Reset(mrfTimeInterval)
case <-p.ctx.Done():
close(p.mrfSaveCh)
saveMRFToDisk(true)
return
case e, ok := <-p.mrfSaveCh:
if !ok {
return
}
var cnt int
mu.Lock()
entries[e.versionID] = e
cnt = len(entries)
mu.Unlock()
if cnt >= cap(p.mrfSaveCh) || len(p.mrfSaveCh) >= int(0.8*float32(cap(p.mrfSaveCh))) {
saveMRFToDisk(true)
}
}
}
}
func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
if p == nil {
return
}
select {
case <-GlobalContext.Done():
return
case p.mrfSaveCh <- entry:
}
}
// save mrf entries to mrf_<uuid>.bin
func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string]MRFReplicateEntry) error {
if len(entries) == 0 {
return nil
}
v := MRFReplicateEntries{
Entries: entries,
Version: mrfMetaVersionV1,
}
data := make([]byte, 4, v.Msgsize()+4)
// Initialize the resync meta header.
binary.LittleEndian.PutUint16(data[0:2], resyncMetaFormat)
binary.LittleEndian.PutUint16(data[2:4], resyncMetaVersion)
buf, err := v.MarshalMsg(data)
if err != nil {
return err
}
configFile := path.Join(replicationMRFDir, mustGetUUID()+".bin")
err = saveConfig(ctx, p.objLayer, configFile, buf)
return err
}
// load mrf entries from disk
func (p *ReplicationPool) loadMRF(fileName string) (re MRFReplicateEntries, e error) {
data, err := readConfig(p.ctx, p.objLayer, fileName)
if err != nil && err != errConfigNotFound {
return re, err
}
if len(data) == 0 {
// Seems to be empty.
return re, nil
}
if len(data) <= 4 {
return re, fmt.Errorf("replication mrf: no data")
}
// Read resync meta header
switch binary.LittleEndian.Uint16(data[0:2]) {
case mrfMetaFormat:
default:
return re, fmt.Errorf("replication mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
}
switch binary.LittleEndian.Uint16(data[2:4]) {
case mrfMetaVersion:
default:
return re, fmt.Errorf("replication mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
}
// OK, parse data.
if _, err = re.UnmarshalMsg(data[4:]); err != nil {
return re, err
}
switch re.Version {
case mrfMetaVersionV1:
default:
return re, fmt.Errorf("unexpected mrf meta version: %d", re.Version)
}
return re, nil
}
func (p *ReplicationPool) processMRF() {
if p == nil || p.objLayer == nil {
return
}
pTimer := time.NewTimer(mrfTimeInterval)
defer pTimer.Stop()
for {
select {
case <-pTimer.C:
// skip healing if all targets are offline
var offlineCnt int
tgts := globalBucketTargetSys.ListTargets(p.ctx, "", "")
for _, tgt := range tgts {
if globalBucketTargetSys.isOffline(tgt.URL()) {
offlineCnt++
}
}
if len(tgts) == offlineCnt {
pTimer.Reset(mrfTimeInterval)
continue
}
objCh := make(chan ObjectInfo)
cctx, cancelFn := context.WithCancel(p.ctx)
if err := p.objLayer.Walk(cctx, minioMetaBucket, replicationMRFDir, objCh, ObjectOptions{}); err != nil {
pTimer.Reset(mrfTimeInterval)
cancelFn()
logger.LogIf(p.ctx, err)
continue
}
for item := range objCh {
if err := p.queueMRFHeal(item.Name); err == nil {
p.objLayer.DeleteObject(p.ctx, minioMetaBucket, item.Name, ObjectOptions{})
}
}
pTimer.Reset(mrfTimeInterval)
cancelFn()
case <-p.ctx.Done():
return
}
}
}
// process sends error logs to the heal channel for an attempt to heal replication.
func (p *ReplicationPool) queueMRFHeal(file string) error {
if p == nil || p.objLayer == nil {
return errServerNotInitialized
}
mrfRec, err := p.loadMRF(file)
if err != nil {
return err
}
for vID, e := range mrfRec.Entries {
oi, err := p.objLayer.GetObjectInfo(p.ctx, e.Bucket, e.Object, ObjectOptions{
VersionID: vID,
})
if err != nil {
continue
}
QueueReplicationHeal(p.ctx, e.Bucket, oi)
}
return nil
}