astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-08-10 00:03:29 +03:00
Code Issues Packages Projects Releases Wiki Activity

Reduce WriteAll allocs (#10810)

Browse Source 
WriteAll saw 127GB allocs in a 5 minute timeframe for 4MiB buffers 
used by `io.CopyBuffer` even if they are pooled.

Since all writers appear to write byte buffers, just send those 
instead and write directly. The files are opened through the `os` 
package so they have no special properties anyway.

This removes the alloc and copy for each operation.

REST sends content length so a precise alloc can be made.
This commit is contained in:
Klaus Post
2020-11-02 16:14:31 -08:00
committed by GitHub
parent 8527f22df1
commit 86e0d272f3
9 changed files with 33 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/xl-storage.go
+12 -10
View File
@@ -1203,7 +1203,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
}
if !lastVersion {
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
}
// Delete the meta file, if there are no more versions the
@@ -1251,7 +1251,7 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
}
}
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), bytes.NewReader(buf))
return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf)
}
func (s *xlStorage) renameLegacyMetadata(volume, path string) error {
@@ -1861,7 +1861,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
return nil
}
func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, reader io.Reader) (err error) {
func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) {
atomic.AddInt32(&s.activeIOCount, 1)
defer func() {
atomic.AddInt32(&s.activeIOCount, -1)
@@ -1873,12 +1873,14 @@ func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, re
}
defer w.Close()
bufp := s.pool.Get().(*[]byte)
defer s.pool.Put(bufp)
_, err = io.CopyBuffer(w, reader, *bufp)
return err
n, err := w.Write(b)
if err != nil {
return err
}
if n != len(b) {
return io.ErrShortWrite
}
return nil
}
// AppendFile - append a byte array at path, if file doesn't exist at
@@ -2311,7 +2313,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
return errFileCorrupt
}
if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), bytes.NewReader(dstBuf)); err != nil {
if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), dstBuf); err != nil {
return err
}