enhance multipart functions to use fsDirent (#1304)

* backend/fs: scanMulitpartDir returns directories only for recursive listing

* backend/fs: enhance multipart functions to use fsDirent

fs-dir-nix.go

@@ -20,6 +20,7 @@ package main
 
 import (
 	"os"
+	"path/filepath"
 	"runtime"
 	"sort"
 	"strings"
@@ -28,7 +29,10 @@ import (
 )
 
 const (
-	// Large enough buffer size for ReadDirent() syscall
+	// readDirentBufSize for syscall.ReadDirent() to hold multiple
+	// directory entries in one buffer. golang source uses 4096 as
+	// buffer size whereas we want 25 times larger to save lots of
+	// entries to avoid multiple syscall.ReadDirent() call.
 	readDirentBufSize = 4096 * 25
 )
 
@@ -65,9 +69,30 @@ func parseDirents(buf []byte) []fsDirent {
 		if name == "." || name == ".." {
 			continue
 		}
+
+		var mode os.FileMode
+		switch dirent.Type {
+		case syscall.DT_BLK, syscall.DT_WHT:
+			mode = os.ModeDevice
+		case syscall.DT_CHR:
+			mode = os.ModeDevice | os.ModeCharDevice
+		case syscall.DT_DIR:
+			mode = os.ModeDir
+		case syscall.DT_FIFO:
+			mode = os.ModeNamedPipe
+		case syscall.DT_LNK:
+			mode = os.ModeSymlink
+		case syscall.DT_REG:
+			mode = 0
+		case syscall.DT_SOCK:
+			mode = os.ModeSocket
+		case syscall.DT_UNKNOWN:
+			mode = 0xffffffff
+		}
+
 		dirents = append(dirents, fsDirent{
-			name:  name,
-			isDir: (dirent.Type == syscall.DT_DIR),
+			name: name,
+			mode: mode,
 		})
 	}
 	return dirents
@@ -91,7 +116,7 @@ func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
 			break
 		}
 		for _, dirent := range parseDirents(buf[:nbuf]) {
-			if dirent.isDir {
+			if dirent.IsDir() {
 				dirent.name += string(os.PathSeparator)
 				dirent.size = 0
 			}
@@ -100,6 +125,46 @@ func readDirAll(readDirPath, entryPrefixMatch string) ([]fsDirent, error) {
 			}
 		}
 	}
-	sort.Sort(byDirentNames(dirents))
+	sort.Sort(byDirentName(dirents))
+	return dirents, nil
+}
+
+// scans the directory dirPath, calling filter() on each directory
+// entry.  Entries for which filter() returns true are stored, lexically
+// sorted using sort.Sort(). If filter is NULL, all entries are selected.
+// If namesOnly is true, dirPath is not appended into entry name.
+func scandir(dirPath string, filter func(fsDirent) bool, namesOnly bool) ([]fsDirent, error) {
+	buf := make([]byte, readDirentBufSize)
+	d, err := os.Open(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	defer d.Close()
+
+	fd := int(d.Fd())
+	dirents := []fsDirent{}
+	for {
+		nbuf, err := syscall.ReadDirent(fd, buf)
+		if err != nil {
+			return nil, err
+		}
+		if nbuf <= 0 {
+			break
+		}
+		for _, dirent := range parseDirents(buf[:nbuf]) {
+			if !namesOnly {
+				dirent.name = filepath.Join(dirPath, dirent.name)
+			}
+			if dirent.IsDir() {
+				dirent.name += string(os.PathSeparator)
+			}
+			if filter == nil || filter(dirent) {
+				dirents = append(dirents, dirent)
+			}
+		}
+	}
+
+	sort.Sort(byDirentName(dirents))
+
 	return dirents, nil
 }