Block change to 1M from 128KiB and extensively use buffer pools.

Following are the benefits of this change.
```
benchmark                           old allocs     new allocs     delta
BenchmarkPutObjectPart5MbFS-4       46172          16383          -64.52%
BenchmarkPutObjectPart10MbFS-4      36849          7223           -80.40%
BenchmarkPutObjectPart25MbFS-4      31699          3865           -87.81%
BenchmarkPutObjectPart50MbFS-4      27577          2481           -91.00%
BenchmarkPutObjectVerySmallFS-4     57             56             -1.75%
BenchmarkPutObject10KbFS-4          57             56             -1.75%
BenchmarkPutObject100KbFS-4         57             56             -1.75%
BenchmarkPutObject1MbFS-4           176            56             -68.18%
BenchmarkPutObject5MbFS-4           720            90             -87.50%
BenchmarkPutObject10MbFS-4          1400           124            -91.14%
BenchmarkPutObject25MbFS-4          3440           261            -92.41%
BenchmarkPutObject50MbFS-4          6841           466            -93.19%
```
This commit is contained in:
Harshavardhana
2016-10-12 17:42:11 -07:00
parent 614a8cf7ad
commit 774e9c8e3a
9 changed files with 137 additions and 32 deletions
+26 -4
View File
@@ -25,6 +25,7 @@ import (
"path"
"sort"
"strings"
"sync"
"github.com/minio/minio/pkg/disk"
"github.com/minio/minio/pkg/mimedb"
@@ -227,6 +228,13 @@ func (fs fsObjects) DeleteBucket(bucket string) error {
/// Object Operations
var getBufPool = sync.Pool{
New: func() interface{} {
b := make([]byte, readSizeV1)
return &b
},
}
// GetObject - get an object.
func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) {
// Verify if bucket is valid.
@@ -266,8 +274,11 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
if length > 0 && bufSize > length {
bufSize = length
}
// Allocate a staging buffer.
buf := make([]byte, int(bufSize))
bufp := getBufPool.Get().(*[]byte)
buf := *bufp
defer getBufPool.Put(bufp)
for {
// Figure out the right size for the buffer.
curLeft := bufSize
@@ -356,7 +367,15 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
}, nil
}
// PutObject - create an object.
var putBufPool = sync.Pool{
New: func() interface{} {
b := make([]byte, readSizeV1)
return &b
},
}
// PutObject - saves an object atomically of size - size bytes.
// With size bytes of '-1' this function reads till EOF.
func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string) (string, error) {
// Verify if bucket is valid.
if !IsValidBucketName(bucket) {
@@ -405,9 +424,12 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
if size > 0 && bufSize > size {
bufSize = size
}
buf := make([]byte, int(bufSize))
bufp := putBufPool.Get().(*[]byte)
buf := *bufp
defer putBufPool.Put(bufp)
teeReader := io.TeeReader(limitDataReader, md5Writer)
bytesWritten, err := fsCreateFile(fs.storage, teeReader, buf, minioMetaBucket, tempObj)
bytesWritten, err := fsCreateFile(fs.storage, teeReader, buf[:bufSize], minioMetaBucket, tempObj)
if err != nil {
fs.storage.DeleteFile(minioMetaBucket, tempObj)
return "", toObjectErr(err, bucket, object)