optimize mkdir calls to avoid base-dir Mkdir attempts (#18021)

Currently we have IOPs of these patterns

```
[OS] os.Mkdir play.min.io:9000 /disk1 2.718µs
[OS] os.Mkdir play.min.io:9000 /disk1/data 2.406µs
[OS] os.Mkdir play.min.io:9000 /disk1/data/.minio.sys 4.068µs
[OS] os.Mkdir play.min.io:9000 /disk1/data/.minio.sys/tmp 2.843µs
[OS] os.Mkdir play.min.io:9000 /disk1/data/.minio.sys/tmp/d89c8ceb-f8d1-4cc6-b483-280f87c4719f 20.152µs
```

It can be seen that we can save quite Nx levels such as
if your drive is mounted at `/disk1/minio` you can simply
skip sending an `Mkdir /disk1/` and `Mkdir /disk1/minio`.

Since they are expected to exist already, this PR adds a way
for us to ignore all paths upto the mount or a directory which
ever has been provided to MinIO setup.
This commit is contained in:
Harshavardhana
2023-09-13 08:14:36 -07:00
committed by GitHub
parent 96fbf18201
commit 8b8be2695f
11 changed files with 60 additions and 50 deletions
+9 -2
View File
@@ -24,6 +24,7 @@ import (
"bytes"
"fmt"
"os"
"strings"
"sync"
"syscall"
"unsafe"
@@ -47,7 +48,13 @@ func access(name string) error {
// directories that MkdirAll creates.
// If path is already a directory, MkdirAll does nothing
// and returns nil.
func osMkdirAll(dirPath string, perm os.FileMode) error {
func osMkdirAll(dirPath string, perm os.FileMode, baseDir string) error {
if baseDir != "" {
if strings.HasPrefix(baseDir, dirPath) {
return nil
}
}
// Slow path: make sure parent exists and then call Mkdir for path.
i := len(dirPath)
for i > 0 && os.IsPathSeparator(dirPath[i-1]) { // Skip trailing path separator.
@@ -61,7 +68,7 @@ func osMkdirAll(dirPath string, perm os.FileMode) error {
if j > 1 {
// Create parent.
if err := osMkdirAll(dirPath[:j-1], perm); err != nil {
if err := osMkdirAll(dirPath[:j-1], perm, baseDir); err != nil {
return err
}
}