Reduce allocations (#17584)

* Reduce allocations

* Add stringsHasPrefixFold which can compare string prefixes, while ignoring case and not allocating.
* Reuse all msgp.Readers
* Reuse metadata buffers when not reading data.

* Make type safe. Make buffer 4K instead of 8.

* Unslice
This commit is contained in:
Klaus Post
2023-07-06 16:02:08 -07:00
committed by GitHub
parent 1bf23374a3
commit ff5988f4e0
21 changed files with 67 additions and 36 deletions
+13
View File
@@ -527,6 +527,10 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
metadataArray := make([]*xlMetaV2, len(disks))
metaFileInfos := make([]FileInfo, len(metadataArray))
metadataShallowVersions := make([][]xlMetaV2ShallowVersion, len(disks))
var v2bufs [][]byte
if !readData {
v2bufs = make([][]byte, len(disks))
}
g := errgroup.WithNErrs(len(disks))
// Read `xl.meta` in parallel across disks.
@@ -540,6 +544,10 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
if err != nil {
return err
}
if !readData {
// Save the buffer so we can reuse it.
v2bufs[index] = rf.Buf
}
var xl xlMetaV2
if err = xl.LoadOrConvert(rf.Buf); err != nil {
@@ -623,6 +631,11 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
}
metaFileInfos[index].DiskMTime = diskMTime
}
if !readData {
for i := range v2bufs {
metaDataPoolPut(v2bufs[i])
}
}
// Return all the metadata.
return metaFileInfos, errs