astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-19 12:10:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add replication capacity metrics support in crawler (#10786)

Browse Source
This commit is contained in:
Ritesh H Shukla
2020-12-07 13:47:48 -08:00
committed by GitHub
parent 6d70f6a4ac
commit 038bcd9079
10 changed files with 235 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/data-crawler.go
+26 -10
View File
@@ -445,16 +445,16 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
lifeCycle: activeLifeCycle,
heal: thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
}
size, err := f.getSize(item)
sizeSummary, err := f.getSize(item)
wait()
if err == errSkipFile {
return nil
}
logger.LogIf(ctx, err)
cache.Size += size
cache.addSizes(sizeSummary)
cache.Objects++
cache.ObjSizes.add(size)
cache.ObjSizes.add(sizeSummary.totalSize)
return nil
})
@@ -673,7 +673,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
}
}
size, err := f.getSize(
sizeSummary, err := f.getSize(
crawlItem{
Path: fileName,
Typ: typ,
@@ -692,9 +692,9 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
return nil
}
logger.LogIf(ctx, err)
cache.Size += size
cache.addSizes(sizeSummary)
cache.Objects++
cache.ObjSizes.add(size)
cache.ObjSizes.add(sizeSummary.totalSize)
return nil
}
err := readDirFn(path.Join(dirStack...), addDir)
@@ -717,7 +717,15 @@ type crawlItem struct {
debug bool
}
type getSizeFn func(item crawlItem) (int64, error)
type sizeSummary struct {
totalSize int64
replicatedSize int64
pendingSize int64
failedSize int64
replicaSize int64
}
type getSizeFn func(item crawlItem) (sizeSummary, error)
// transformMetaDir will transform a directory to prefix/file.ext
func (i *crawlItem) transformMetaDir() {
@@ -910,7 +918,7 @@ func (i *crawlItem) objectPath() string {
}
// healReplication will heal a scanned item that has failed replication.
func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta) {
func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta, sizeS *sizeSummary) {
if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() {
//heal delete marker replication failure or versioned delete replication failure
if meta.oi.ReplicationStatus == replication.Pending ||
@@ -920,9 +928,17 @@ func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta act
return
}
}
if meta.oi.ReplicationStatus == replication.Pending ||
meta.oi.ReplicationStatus == replication.Failed {
switch meta.oi.ReplicationStatus {
case replication.Pending:
sizeS.pendingSize += meta.oi.Size
globalReplicationState.queueReplicaTask(meta.oi)
case replication.Failed:
sizeS.failedSize += meta.oi.Size
globalReplicationState.queueReplicaTask(meta.oi)
case replication.Complete:
sizeS.replicatedSize += meta.oi.Size
case replication.Replica:
sizeS.replicaSize += meta.oi.Size
}
}