[feat]: use DSYNC for xl.meta writes and NOATIME for reads (#11615)

Instead of using O_SYNC, we are better off using O_DSYNC instead since we are only ever interested in data to be persisted to disk not the associated filesystem metadata. For reads we ask customers to turn off noatime, but instead we can proactively use O_NOATIME flag to avoid atime updates upon reads.
2026-10-08 18:55:58 +03:00 · 2021-02-24 00:14:16 -08:00
parent 14aef52004
commit b517c791e9
14 changed files with 278 additions and 74 deletions
@@ -264,9 +264,10 @@ func newXLStorage(ep Endpoint) (*xlStorage, error) {
 				return &b
 			},
 		},
-		globalSync: env.Get(config.EnvFSOSync, config.EnableOff) == config.EnableOn,
-		ctx:        GlobalContext,
-		rootDisk:   rootDisk,
+		globalSync:           env.Get(config.EnvFSOSync, config.EnableOff) == config.EnableOn,
+		ctx:                  GlobalContext,
+		rootDisk:             rootDisk,
+		readODirectSupported: true,
 	}

 	// Create all necessary bucket folders if possible.
@@ -382,7 +383,7 @@ func (s *xlStorage) CrawlAndGetDataUsage(ctx context.Context, cache dataUsageCac
 			return sizeSummary{}, errSkipFile
 		}

-		buf, err := ioutil.ReadFile(item.Path)
+		buf, err := xioutil.ReadFile(item.Path)
 		if err != nil {
 			if intDataUpdateTracker.debug {
 				console.Debugf(color.Green("crawlBucket:")+" object path missing: %v: %w\n", item.Path, err)
@@ -539,7 +540,7 @@ func (s *xlStorage) GetDiskID() (string, error) {
 		return diskID, nil
 	}

-	b, err := ioutil.ReadFile(formatFile)
+	b, err := xioutil.ReadFile(formatFile)
 	if err != nil {
 		// If the disk is still not initialized.
 		if osIsNotExist(err) {
@@ -827,7 +828,7 @@ func (s *xlStorage) WalkVersions(ctx context.Context, volume, dirPath, marker st
 					},
 				}
 			} else {
-				xlMetaBuf, err := ioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, xlStorageFormatFile))
+				xlMetaBuf, err := xioutil.ReadFile(pathJoin(volumeDir, walkResult.entry, xlStorageFormatFile))
 				if err != nil {
 					continue
 				}
@@ -1150,9 +1151,9 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirectIO bool) (buf []byte, err error) {
 	var f *os.File
 	if requireDirectIO {
-		f, err = disk.OpenFileDirectIO(filePath, os.O_RDONLY, 0666)
+		f, err = disk.OpenFileDirectIO(filePath, readMode, 0666)
 	} else {
-		f, err = os.Open(filePath)
+		f, err = os.OpenFile(filePath, readMode, 0)
 	}
 	if err != nil {
 		if osIsNotExist(err) {
@@ -1175,6 +1176,13 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string, requireDirect
 		} else if isSysErrTooManyFiles(err) {
 			return nil, errTooManyOpenFiles
 		} else if isSysErrInvalidArg(err) {
+			st, _ := os.Lstat(filePath)
+			if st != nil && st.IsDir() {
+				// Linux returns InvalidArg for directory O_DIRECT
+				// we need to keep this fallback code to return correct
+				// errors upwards.
+				return nil, errFileNotFound
+			}
 			return nil, errUnsupportedDisk
 		}
 		return nil, err
@@ -1328,42 +1336,23 @@ func (s *xlStorage) openFile(volume, path string, mode int) (f *os.File, err err
 		return nil, err
 	}

-	// Stat a volume entry.
-	_, err = os.Lstat(volumeDir)
-	if err != nil {
-		if osIsNotExist(err) {
-			return nil, errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return nil, errFaultyDisk
-		}
-		return nil, err
-	}
-
 	filePath := pathJoin(volumeDir, path)
 	if err = checkPathLength(filePath); err != nil {
 		return nil, err
 	}

-	// Verify if the file already exists and is not of regular type.
-	var st os.FileInfo
-	if st, err = os.Lstat(filePath); err == nil {
-		if !st.Mode().IsRegular() {
-			return nil, errIsNotRegular
-		}
-	} else {
-		// Create top level directories if they don't exist.
-		// with mode 0777 mkdir honors system umask.
-		if err = mkdirAll(pathutil.Dir(filePath), 0777); err != nil {
-			return nil, err
-		}
+	// Create top level directories if they don't exist.
+	// with mode 0777 mkdir honors system umask.
+	if err = mkdirAll(pathutil.Dir(filePath), 0777); err != nil {
+		return nil, err
 	}

-	w, err := os.OpenFile(filePath, mode, 0666)
+	w, err := os.OpenFile(filePath, mode|writeMode, 0666)
 	if err != nil {
 		// File path cannot be verified since one of the parents is a file.
 		switch {
-		case isSysErrNotDir(err):
-			return nil, errFileAccessDenied
+		case isSysErrIsDir(err):
+			return nil, errIsNotRegular
 		case osIsPermission(err):
 			return nil, errFileAccessDenied
 		case isSysErrIO(err):
@@ -1668,19 +1657,21 @@ func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, b
 		atomic.AddInt32(&s.activeIOCount, -1)
 	}()

-	w, err := s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_WRONLY)
+	w, err := s.openFile(volume, path, os.O_CREATE|os.O_WRONLY)
 	if err != nil {
 		return err
 	}
-
 	defer w.Close()
+
 	n, err := w.Write(b)
 	if err != nil {
 		return err
 	}
+
 	if n != len(b) {
 		return io.ErrShortWrite
 	}
+
 	return nil
 }

@@ -1692,19 +1683,38 @@ func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string,
 		atomic.AddInt32(&s.activeIOCount, -1)
 	}()

-	var w *os.File
-	// Create file if not found. Not doing O_DIRECT here to avoid the code that does buffer aligned writes.
-	// AppendFile() is only used by healing code to heal objects written in old format.
-	w, err = s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_APPEND|os.O_WRONLY)
+	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
 		return err
 	}

-	if _, err = w.Write(buf); err != nil {
+	// Stat a volume entry.
+	if _, err = os.Lstat(volumeDir); err != nil {
+		if osIsNotExist(err) {
+			return errVolumeNotFound
+		}
 		return err
 	}

-	return w.Close()
+	var w *os.File
+	// Create file if not found. Not doing O_DIRECT here to avoid the code that does buffer aligned writes.
+	// AppendFile() is only used by healing code to heal objects written in old format.
+	w, err = s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
+	if err != nil {
+		return err
+	}
+	defer w.Close()
+
+	n, err := w.Write(buf)
+	if err != nil {
+		return err
+	}
+
+	if n != len(buf) {
+		return io.ErrShortWrite
+	}
+
+	return nil
 }

 // CheckParts check if path has necessary parts available.
@@ -2000,7 +2010,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
 		return err
 	}

-	srcBuf, err := ioutil.ReadFile(srcFilePath)
+	srcBuf, err := xioutil.ReadFile(srcFilePath)
 	if err != nil {
 		return osErrToFileErr(err)
 	}
@@ -2010,7 +2020,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
 		return err
 	}

-	dstBuf, err := ioutil.ReadFile(dstFilePath)
+	dstBuf, err := xioutil.ReadFile(dstFilePath)
 	if err != nil {
 		if !osIsNotExist(err) {
 			return osErrToFileErr(err)
@@ -2021,7 +2031,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath, dataDir,
 			return err
 		}
 		if err == nil {
-			dstBuf, err = ioutil.ReadFile(dstFilePath)
+			dstBuf, err = xioutil.ReadFile(dstFilePath)
 			if err != nil && !osIsNotExist(err) {
 				return osErrToFileErr(err)
 			}