cache disk info to avoid repeated calls (#9682)

This value is requested on every upload when there are multiple zones.

Since this will result in an RPC call to every remote disk this scales 
quite badly in a distributed setup. Load every 1second interval.

2 servers, localhost only. In large distributed setups much bigger 
gains can be expected.

```
Operations: 21743 -> 22454
* Average: +3.28% (+0.0 MiB/s) throughput, +3.28% (+11.9) obj/s
* Fastest: +3.37% (+0.0 MiB/s) throughput, +3.37% (+13.0) obj/s
* 50% Median: +3.03% (+0.0 MiB/s) throughput, +3.03% (+11.2) obj/s
* Slowest: +8.03% (+0.0 MiB/s) throughput, +8.03% (+22.8) obj/s
```

For easy management of this a generic helper has been added.
This commit is contained in:
Klaus Post
2020-05-26 12:52:24 -07:00
committed by GitHub
parent de6c286258
commit 95814359bd
2 changed files with 73 additions and 0 deletions
+16
View File
@@ -95,6 +95,8 @@ type xlSets struct {
// Distribution algorithm of choice.
distributionAlgo string
disksStorageInfoCache timedValue
// Merge tree walk
pool *MergeWalkPool
poolSplunk *MergeWalkPool
@@ -368,7 +370,21 @@ func (s *xlSets) NewNSLock(ctx context.Context, bucket string, objects ...string
}
// StorageInfo - combines output of StorageInfo across all erasure coded object sets.
// Caches values for 1 second.
func (s *xlSets) StorageInfo(ctx context.Context, local bool) StorageInfo {
s.disksStorageInfoCache.Once.Do(func() {
s.disksStorageInfoCache.TTL = time.Second
s.disksStorageInfoCache.Update = func() (interface{}, error) {
return s.storageInfo(ctx, local), nil
}
})
v, _ := s.disksStorageInfoCache.Get()
return v.(StorageInfo)
}
// storageInfo - combines output of StorageInfo across all erasure coded object sets.
// Use StorageInfo for a cached version.
func (s *xlSets) storageInfo(ctx context.Context, local bool) StorageInfo {
var storageInfo StorageInfo
storageInfos := make([]StorageInfo, len(s.sets))