use fadvise to control Linux page-cache (#13312)

This PR brings two optimizations mainly
for page-cache build-up and how to avoid
getting OOM killed in the process. Although
these memories are reclaimable Linux is not
fast enough to reclaim them as needed on a
very busy system. fadvise is a system call
implemented in Linux to advise page-cache to
avoid overload as we get significant amount
of requests on the server.

- FADV_SEQUENTIAL tells that all I/O from now
  is going to be sequential, allowing for more
  resposive throughput.

- FADV_NOREUSE tells kernel to start removing
  things for this 'fd' from page-cache.
This commit is contained in:
Harshavardhana
2021-09-28 10:02:56 -07:00
committed by GitHub
parent dd5804c10e
commit 38027c8f52
6 changed files with 73 additions and 1 deletions
+12
View File
@@ -28,6 +28,7 @@ import (
"syscall"
"unsafe"
"github.com/minio/minio/internal/disk"
"golang.org/x/sys/unix"
)
@@ -110,6 +111,11 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
}
return osErrToFileErr(err)
}
if err := disk.Fadvise(f, disk.FadvSequential); err != nil {
return err
}
defer disk.Fadvise(f, disk.FadvNoReuse)
defer f.Close()
bufp := direntPool.Get().(*[]byte)
@@ -185,6 +191,12 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
if err != nil {
return nil, osErrToFileErr(err)
}
if err := disk.Fadvise(f, disk.FadvSequential); err != nil {
return nil, err
}
defer disk.Fadvise(f, disk.FadvNoReuse)
defer f.Close()
bufp := direntPool.Get().(*[]byte)