perf: websocket grid connectivity for all internode communication (#18461)

This PR adds a WebSocket grid feature that allows servers to communicate via 
a single two-way connection.

There are two request types:

* Single requests, which are `[]byte => ([]byte, error)`. This is for efficient small
  roundtrips with small payloads.

* Streaming requests which are `[]byte, chan []byte => chan []byte (and error)`,
  which allows for different combinations of full two-way streams with an initial payload.

Only a single stream is created between two machines - and there is, as such, no
server/client relation since both sides can initiate and handle requests. Which server
initiates the request is decided deterministically on the server names.

Requests are made through a mux client and server, which handles message
passing, congestion, cancelation, timeouts, etc.

If a connection is lost, all requests are canceled, and the calling server will try
to reconnect. Registered handlers can operate directly on byte 
slices or use a higher-level generics abstraction.

There is no versioning of handlers/clients, and incompatible changes should
be handled by adding new handlers.

The request path can be changed to a new one for any protocol changes.

First, all servers create a "Manager." The manager must know its address 
as well as all remote addresses. This will manage all connections.
To get a connection to any remote, ask the manager to provide it given
the remote address using.

```
func (m *Manager) Connection(host string) *Connection
```

All serverside handlers must also be registered on the manager. This will
make sure that all incoming requests are served. The number of in-flight 
requests and responses must also be given for streaming requests.

The "Connection" returned manages the mux-clients. Requests issued
to the connection will be sent to the remote.

* `func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error)`
   performs a single request and returns the result. Any deadline provided on the request is
   forwarded to the server, and canceling the context will make the function return at once.

* `func (c *Connection) NewStream(ctx context.Context, h HandlerID, payload []byte) (st *Stream, err error)`
   will initiate a remote call and send the initial payload.

```Go
// A Stream is a two-way stream.
// All responses *must* be read by the caller.
// If the call is canceled through the context,
//The appropriate error will be returned.
type Stream struct {
	// Responses from the remote server.
	// Channel will be closed after an error or when the remote closes.
	// All responses *must* be read by the caller until either an error is returned or the channel is closed.
	// Canceling the context will cause the context cancellation error to be returned.
	Responses <-chan Response

	// Requests sent to the server.
	// If the handler is defined with 0 incoming capacity this will be nil.
	// Channel *must* be closed to signal the end of the stream.
	// If the request context is canceled, the stream will no longer process requests.
	Requests chan<- []byte
}

type Response struct {
	Msg []byte
	Err error
}
```

There are generic versions of the server/client handlers that allow the use of type
safe implementations for data types that support msgpack marshal/unmarshal.

cmd/storage-rest-server.go

@@ -33,8 +33,10 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
+	"github.com/minio/minio/internal/grid"
 	"github.com/tinylib/msgp/msgp"
 
 	jwtreq "github.com/golang-jwt/jwt/v4/request"
@@ -116,7 +118,7 @@ func storageServerRequestValidate(r *http.Request) error {
 	return nil
 }
 
-// IsValid - To authenticate and verify the time difference.
+// IsAuthValid - To authenticate and verify the time difference.
 func (s *storageRESTServer) IsAuthValid(w http.ResponseWriter, r *http.Request) bool {
 	if s.storage == nil {
 		s.writeErrorResponse(w, errDiskNotFound)
@@ -165,56 +167,63 @@ func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool
 	return true
 }
 
+// checkID - check if the disk-id in the request corresponds to the underlying disk.
+func (s *storageRESTServer) checkID(wantID string) bool {
+	if s.storage == nil {
+		return false
+	}
+	if wantID == "" {
+		// Request sent empty disk-id, we allow the request
+		// as the peer might be coming up and trying to read format.json
+		// or create format.json
+		return true
+	}
+
+	storedDiskID, err := s.storage.GetDiskID()
+	if err != nil {
+		return false
+	}
+
+	return wantID == storedDiskID
+}
+
 // HealthHandler handler checks if disk is stale
 func (s *storageRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {
 	s.IsValid(w, r)
 }
 
+// DiskInfo types.
+// DiskInfo.Metrics elements are shared, so we cannot reuse.
+var storageDiskInfoHandler = grid.NewSingleHandler[*grid.MSS, *DiskInfo](grid.HandlerDiskInfo, grid.NewMSS, func() *DiskInfo { return &DiskInfo{} }).WithSharedResponse()
+
 // DiskInfoHandler - returns disk info.
-func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsAuthValid(w, r) {
-		return
+func (s *storageRESTServer) DiskInfoHandler(params *grid.MSS) (*DiskInfo, *grid.RemoteErr) {
+	if !s.checkID(params.Get(storageRESTDiskID)) {
+		return nil, grid.NewRemoteErr(errDiskNotFound)
 	}
-	info, err := s.storage.DiskInfo(r.Context(), r.Form.Get(storageRESTMetrics) == "true")
+	withMetrics := params.Get(storageRESTMetrics) == "true"
+	info, err := s.storage.DiskInfo(context.Background(), withMetrics)
 	if err != nil {
 		info.Error = err.Error()
 	}
-	logger.LogIf(r.Context(), msgp.Encode(w, &info))
+	info.Scanning = s.storage != nil && s.storage.storage != nil && atomic.LoadInt32(&s.storage.storage.scanning) > 0
+	return &info, nil
 }
 
-func (s *storageRESTServer) NSScannerHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
+// scanner rpc handler.
+var storageNSScannerHandler = grid.NewStream[*nsScannerOptions, grid.NoPayload, *nsScannerResp](grid.HandlerNSScanner,
+	func() *nsScannerOptions { return &nsScannerOptions{} },
+	nil,
+	func() *nsScannerResp { return &nsScannerResp{} })
+
+func (s *storageRESTServer) NSScannerHandler(ctx context.Context, params *nsScannerOptions, out chan<- *nsScannerResp) *grid.RemoteErr {
+	if !s.checkID(params.DiskID) {
+		return grid.NewRemoteErr(errDiskNotFound)
 	}
-
-	scanMode, err := strconv.Atoi(r.Form.Get(storageRESTScanMode))
-	if err != nil {
-		logger.LogIf(r.Context(), err)
-		s.writeErrorResponse(w, err)
-		return
+	if params.Cache == nil {
+		return grid.NewRemoteErrString("NSScannerHandler: provided cache is nil")
 	}
 
-	setEventStreamHeaders(w)
-
-	var cache dataUsageCache
-	err = cache.deserialize(r.Body)
-	if err != nil {
-		logger.LogIf(r.Context(), err)
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	ctx, cancel := context.WithCancel(r.Context())
-	defer cancel()
-	resp := streamHTTPResponse(w)
-	defer func() {
-		if r := recover(); r != nil {
-			debug.PrintStack()
-			resp.CloseWithError(fmt.Errorf("panic: %v", r))
-		}
-	}()
-	respW := msgp.NewWriter(resp)
-
 	// Collect updates, stream them before the full cache is sent.
 	updates := make(chan dataUsageEntry, 1)
 	var wg sync.WaitGroup
@@ -222,36 +231,21 @@ func (s *storageRESTServer) NSScannerHandler(w http.ResponseWriter, r *http.Requ
 	go func() {
 		defer wg.Done()
 		for update := range updates {
-			// Write true bool to indicate update.
-			var err error
-			if err = respW.WriteBool(true); err == nil {
-				err = update.EncodeMsg(respW)
-			}
-			respW.Flush()
-			if err != nil {
-				cancel()
-				resp.CloseWithError(err)
-				return
-			}
+			resp := storageNSScannerHandler.NewResponse()
+			resp.Update = &update
+			out <- resp
 		}
 	}()
-	usageInfo, err := s.storage.NSScanner(ctx, cache, updates, madmin.HealScanMode(scanMode))
-	if err != nil {
-		respW.Flush()
-		resp.CloseWithError(err)
-		return
-	}
-
-	// Write false bool to indicate we finished.
+	ui, err := s.storage.NSScanner(ctx, *params.Cache, updates, madmin.HealScanMode(params.ScanMode))
 	wg.Wait()
-	if err = respW.WriteBool(false); err == nil {
-		err = usageInfo.EncodeMsg(respW)
-	}
 	if err != nil {
-		resp.CloseWithError(err)
-		return
+		return grid.NewRemoteErr(err)
 	}
-	resp.CloseWithError(respW.Flush())
+	// Send final response.
+	resp := storageNSScannerHandler.NewResponse()
+	resp.Final = &ui
+	out <- resp
+	return nil
 }
 
 // MakeVolHandler - make a volume.
@@ -291,21 +285,22 @@ func (s *storageRESTServer) ListVolsHandler(w http.ResponseWriter, r *http.Reque
 	logger.LogIf(r.Context(), msgp.Encode(w, VolsInfo(infos)))
 }
 
+// statvol types.
+var storageStatVolHandler = grid.NewSingleHandler[*grid.MSS, *VolInfo](grid.HandlerStatVol, grid.NewMSS, func() *VolInfo { return &VolInfo{} })
+
 // StatVolHandler - stat a volume.
-func (s *storageRESTServer) StatVolHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
+func (s *storageRESTServer) StatVolHandler(params *grid.MSS) (*VolInfo, *grid.RemoteErr) {
+	if !s.checkID(params.Get(storageRESTDiskID)) {
+		return nil, grid.NewRemoteErr(errDiskNotFound)
 	}
-	volume := r.Form.Get(storageRESTVolume)
-	info, err := s.storage.StatVol(r.Context(), volume)
+	info, err := s.storage.StatVol(context.Background(), params.Get(storageRESTVolume))
 	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
+		return nil, grid.NewRemoteErr(err)
 	}
-	logger.LogIf(r.Context(), msgp.Encode(w, &info))
+	return &info, nil
 }
 
-// DeleteVolumeHandler - delete a volume.
+// DeleteVolHandler - delete a volume.
 func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		return
@@ -357,37 +352,48 @@ func (s *storageRESTServer) CreateFileHandler(w http.ResponseWriter, r *http.Req
 	done(s.storage.CreateFile(r.Context(), volume, filePath, int64(fileSize), body))
 }
 
-// DeleteVersion delete updated metadata.
-func (s *storageRESTServer) DeleteVersionHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
-	}
-	volume := r.Form.Get(storageRESTVolume)
-	filePath := r.Form.Get(storageRESTFilePath)
-	forceDelMarker, err := strconv.ParseBool(r.Form.Get(storageRESTForceDelMarker))
-	if err != nil {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
-	}
+var storageDeleteVersionHandler = grid.NewSingleHandler[*DeleteVersionHandlerParams, grid.NoPayload](grid.HandlerDeleteVersion, func() *DeleteVersionHandlerParams {
+	return &DeleteVersionHandlerParams{}
+}, grid.NewNoPayload)
 
-	if r.ContentLength < 0 {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
+// DeleteVersionHandler delete updated metadata.
+func (s *storageRESTServer) DeleteVersionHandler(p *DeleteVersionHandlerParams) (np grid.NoPayload, gerr *grid.RemoteErr) {
+	if !s.checkID(p.DiskID) {
+		return np, grid.NewRemoteErr(errDiskNotFound)
 	}
+	volume := p.Volume
+	filePath := p.FilePath
+	forceDelMarker := p.ForceDelMarker
 
-	var fi FileInfo
-	if err := msgp.Decode(r.Body, &fi); err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	err = s.storage.DeleteVersion(r.Context(), volume, filePath, fi, forceDelMarker)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-	}
+	err := s.storage.DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker)
+	return np, grid.NewRemoteErr(err)
 }
 
-// ReadVersion read metadata of versionID
+var storageReadVersionHandler = grid.NewSingleHandler[*grid.MSS, *FileInfo](grid.HandlerReadVersion, grid.NewMSS, func() *FileInfo {
+	return &FileInfo{}
+})
+
+// ReadVersionHandlerWS read metadata of versionID
+func (s *storageRESTServer) ReadVersionHandlerWS(params *grid.MSS) (*FileInfo, *grid.RemoteErr) {
+	if !s.checkID(params.Get(storageRESTDiskID)) {
+		return nil, grid.NewRemoteErr(errDiskNotFound)
+	}
+	volume := params.Get(storageRESTVolume)
+	filePath := params.Get(storageRESTFilePath)
+	versionID := params.Get(storageRESTVersionID)
+	readData, err := strconv.ParseBool(params.Get(storageRESTReadData))
+	if err != nil {
+		return nil, grid.NewRemoteErr(err)
+	}
+
+	fi, err := s.storage.ReadVersion(context.Background(), volume, filePath, versionID, readData)
+	if err != nil {
+		return nil, grid.NewRemoteErr(err)
+	}
+	return &fi, nil
+}
+
+// ReadVersionHandler read metadata of versionID
 func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		return
@@ -410,55 +416,35 @@ func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Re
 	logger.LogIf(r.Context(), msgp.Encode(w, &fi))
 }
 
-// WriteMetadata write new updated metadata.
-func (s *storageRESTServer) WriteMetadataHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
-	}
-	volume := r.Form.Get(storageRESTVolume)
-	filePath := r.Form.Get(storageRESTFilePath)
+var storageWriteMetadataHandler = grid.NewSingleHandler[*MetadataHandlerParams, grid.NoPayload](grid.HandlerWriteMetadata, func() *MetadataHandlerParams {
+	return &MetadataHandlerParams{}
+}, grid.NewNoPayload)
 
-	if r.ContentLength < 0 {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
+// WriteMetadataHandler rpc handler to write new updated metadata.
+func (s *storageRESTServer) WriteMetadataHandler(p *MetadataHandlerParams) (np grid.NoPayload, gerr *grid.RemoteErr) {
+	if !s.checkID(p.DiskID) {
+		return grid.NewNPErr(errDiskNotFound)
 	}
+	volume := p.Volume
+	filePath := p.FilePath
 
-	var fi FileInfo
-	if err := msgp.Decode(r.Body, &fi); err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	err := s.storage.WriteMetadata(r.Context(), volume, filePath, fi)
-	if err != nil {
-		s.writeErrorResponse(w, err)
-	}
+	err := s.storage.WriteMetadata(context.Background(), volume, filePath, p.FI)
+	return np, grid.NewRemoteErr(err)
 }
 
-// UpdateMetadata update new updated metadata.
-func (s *storageRESTServer) UpdateMetadataHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
-	}
-	volume := r.Form.Get(storageRESTVolume)
-	filePath := r.Form.Get(storageRESTFilePath)
-	noPersistence := r.Form.Get(storageRESTNoPersistence) == "true"
+var storageUpdateMetadataHandler = grid.NewSingleHandler[*MetadataHandlerParams, grid.NoPayload](grid.HandlerUpdateMetadata, func() *MetadataHandlerParams {
+	return &MetadataHandlerParams{}
+}, grid.NewNoPayload)
 
-	if r.ContentLength < 0 {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
+// UpdateMetadataHandler update new updated metadata.
+func (s *storageRESTServer) UpdateMetadataHandler(p *MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
+	if !s.checkID(p.DiskID) {
+		return grid.NewNPErr(errDiskNotFound)
 	}
+	volume := p.Volume
+	filePath := p.FilePath
 
-	var fi FileInfo
-	if err := msgp.Decode(r.Body, &fi); err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	err := s.storage.UpdateMetadata(r.Context(), volume, filePath, fi, UpdateMetadataOpts{NoPersistence: noPersistence})
-	if err != nil {
-		s.writeErrorResponse(w, err)
-	}
+	return grid.NewNPErr(s.storage.UpdateMetadata(context.Background(), volume, filePath, p.FI, p.UpdateOpts))
 }
 
 // WriteAllHandler - write to file all content.
@@ -485,28 +471,18 @@ func (s *storageRESTServer) WriteAllHandler(w http.ResponseWriter, r *http.Reque
 	}
 }
 
+var storageCheckPartsHandler = grid.NewSingleHandler[*CheckPartsHandlerParams, grid.NoPayload](grid.HandlerCheckParts, func() *CheckPartsHandlerParams {
+	return &CheckPartsHandlerParams{}
+}, grid.NewNoPayload)
+
 // CheckPartsHandler - check if a file metadata exists.
-func (s *storageRESTServer) CheckPartsHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
-	}
-	volume := r.Form.Get(storageRESTVolume)
-	filePath := r.Form.Get(storageRESTFilePath)
-
-	if r.ContentLength < 0 {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
-	}
-
-	var fi FileInfo
-	if err := msgp.Decode(r.Body, &fi); err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	if err := s.storage.CheckParts(r.Context(), volume, filePath, fi); err != nil {
-		s.writeErrorResponse(w, err)
+func (s *storageRESTServer) CheckPartsHandler(p *CheckPartsHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
+	if !s.checkID(p.DiskID) {
+		return grid.NewNPErr(errDiskNotFound)
 	}
+	volume := p.Volume
+	filePath := p.FilePath
+	return grid.NewNPErr(s.storage.CheckParts(context.Background(), volume, filePath, p.FI))
 }
 
 // ReadAllHandler - read all the contents of a file.
@@ -550,6 +526,30 @@ func (s *storageRESTServer) ReadXLHandler(w http.ResponseWriter, r *http.Request
 	logger.LogIf(r.Context(), msgp.Encode(w, &rf))
 }
 
+var storageReadXLHandler = grid.NewSingleHandler[*grid.MSS, *RawFileInfo](grid.HandlerReadXL, grid.NewMSS, func() *RawFileInfo {
+	return &RawFileInfo{}
+})
+
+// ReadXLHandlerWS - read xl.meta for an object at path.
+func (s *storageRESTServer) ReadXLHandlerWS(params *grid.MSS) (*RawFileInfo, *grid.RemoteErr) {
+	if !s.checkID(params.Get(storageRESTDiskID)) {
+		return nil, grid.NewRemoteErr(errDiskNotFound)
+	}
+	volume := params.Get(storageRESTVolume)
+	filePath := params.Get(storageRESTFilePath)
+	readData, err := strconv.ParseBool(params.Get(storageRESTReadData))
+	if err != nil {
+		return nil, grid.NewRemoteErr(err)
+	}
+
+	rf, err := s.storage.ReadXL(context.Background(), volume, filePath, readData)
+	if err != nil {
+		return nil, grid.NewRemoteErr(err)
+	}
+
+	return &rf, nil
+}
+
 // ReadFileHandler - read section of a file.
 func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -593,7 +593,7 @@ func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Reque
 	w.Write(buf)
 }
 
-// ReadFileHandler - read section of a file.
+// ReadFileStreamHandler - read section of a file.
 func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		return
@@ -666,30 +666,16 @@ func (s *storageRESTServer) ListDirHandler(w http.ResponseWriter, r *http.Reques
 	gob.NewEncoder(w).Encode(&entries)
 }
 
+var storageDeleteFileHandler = grid.NewSingleHandler[*DeleteFileHandlerParams, grid.NoPayload](grid.HandlerDeleteFile, func() *DeleteFileHandlerParams {
+	return &DeleteFileHandlerParams{}
+}, grid.NewNoPayload)
+
 // DeleteFileHandler - delete a file.
-func (s *storageRESTServer) DeleteFileHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
-	}
-	volume := r.Form.Get(storageRESTVolume)
-	filePath := r.Form.Get(storageRESTFilePath)
-	recursive, err := strconv.ParseBool(r.Form.Get(storageRESTRecursive))
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-	force, err := strconv.ParseBool(r.Form.Get(storageRESTForceDelete))
-	if err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-	err = s.storage.Delete(r.Context(), volume, filePath, DeleteOptions{
-		Recursive: recursive,
-		Force:     force,
-	})
-	if err != nil {
-		s.writeErrorResponse(w, err)
+func (s *storageRESTServer) DeleteFileHandler(p *DeleteFileHandlerParams) (grid.NoPayload, *grid.RemoteErr) {
+	if !s.checkID(p.DiskID) {
+		return grid.NewNPErr(errDiskNotFound)
 	}
+	return grid.NewNPErr(s.storage.Delete(context.Background(), p.Volume, p.FilePath, p.Opts))
 }
 
 // DeleteVersionsErrsResp - collection of delete errors
@@ -737,48 +723,23 @@ func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http
 	encoder.Encode(dErrsResp)
 }
 
-// RenameDataResp - RenameData()'s response.
-type RenameDataResp struct {
-	Signature uint64
-	Err       error
-}
+var storageRenameDataHandler = grid.NewSingleHandler[*RenameDataHandlerParams, *RenameDataResp](grid.HandlerRenamedata, func() *RenameDataHandlerParams {
+	return &RenameDataHandlerParams{}
+}, func() *RenameDataResp {
+	return &RenameDataResp{}
+})
 
 // RenameDataHandler - renames a meta object and data dir to destination.
-func (s *storageRESTServer) RenameDataHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
-		return
+func (s *storageRESTServer) RenameDataHandler(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) {
+	if !s.checkID(p.DiskID) {
+		return nil, grid.NewRemoteErr(errDiskNotFound)
 	}
 
-	srcVolume := r.Form.Get(storageRESTSrcVolume)
-	srcFilePath := r.Form.Get(storageRESTSrcPath)
-	dstVolume := r.Form.Get(storageRESTDstVolume)
-	dstFilePath := r.Form.Get(storageRESTDstPath)
-
-	if r.ContentLength < 0 {
-		s.writeErrorResponse(w, errInvalidArgument)
-		return
-	}
-
-	var fi FileInfo
-	if err := msgp.Decode(r.Body, &fi); err != nil {
-		s.writeErrorResponse(w, err)
-		return
-	}
-
-	setEventStreamHeaders(w)
-	encoder := gob.NewEncoder(w)
-	done := keepHTTPResponseAlive(w)
-
-	sign, err := s.storage.RenameData(r.Context(), srcVolume, srcFilePath, fi, dstVolume, dstFilePath)
-	done(nil)
-
+	sign, err := s.storage.RenameData(context.Background(), p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath)
 	resp := &RenameDataResp{
 		Signature: sign,
 	}
-	if err != nil {
-		resp.Err = StorageErr(err.Error())
-	}
-	encoder.Encode(resp)
+	return resp, grid.NewRemoteErr(err)
 }
 
 // RenameFileHandler - rename a file.
@@ -1349,8 +1310,8 @@ func (s *storageRESTServer) ReadMultiple(w http.ResponseWriter, r *http.Request)
 	rw.CloseWithError(err)
 }
 
-// registerStorageRPCRouter - register storage rpc router.
-func registerStorageRESTHandlers(router *mux.Router, endpointServerPools EndpointServerPools) {
+// registerStorageRESTHandlers - register storage rpc router.
+func registerStorageRESTHandlers(router *mux.Router, endpointServerPools EndpointServerPools, gm *grid.Manager) {
 	storageDisks := make([][]*xlStorage, len(endpointServerPools))
 	for poolIdx, ep := range endpointServerPools {
 		storageDisks[poolIdx] = make([]*xlStorage, len(ep.Endpoints))
@@ -1394,38 +1355,44 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
 
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodHealth).HandlerFunc(h(server.HealthHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(h(server.DiskInfoHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodNSScanner).HandlerFunc(h(server.NSScannerHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(h(server.MakeVolHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVolBulk).HandlerFunc(h(server.MakeVolBulkHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatVol).HandlerFunc(h(server.StatVolHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVol).HandlerFunc(h(server.DeleteVolHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListVols).HandlerFunc(h(server.ListVolsHandler))
 
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodAppendFile).HandlerFunc(h(server.AppendFileHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteAll).HandlerFunc(h(server.WriteAllHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteMetadata).HandlerFunc(h(server.WriteMetadataHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodUpdateMetadata).HandlerFunc(h(server.UpdateMetadataHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersion).HandlerFunc(h(server.DeleteVersionHandler))
+
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadVersion).HandlerFunc(h(server.ReadVersionHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadXL).HandlerFunc(h(server.ReadXLHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameData).HandlerFunc(h(server.RenameDataHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCreateFile).HandlerFunc(h(server.CreateFileHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCheckParts).HandlerFunc(h(server.CheckPartsHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadAll).HandlerFunc(h(server.ReadAllHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFile).HandlerFunc(h(server.ReadFileHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFileStream).HandlerFunc(h(server.ReadFileStreamHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListDir).HandlerFunc(h(server.ListDirHandler))
 
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersions).HandlerFunc(h(server.DeleteVersionsHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFile).HandlerFunc(h(server.DeleteFileHandler))
-
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameFile).HandlerFunc(h(server.RenameFileHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(h(server.VerifyFileHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(h(server.WalkDirHandler))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(h(server.StatInfoFile))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(h(server.ReadMultiple))
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCleanAbandoned).HandlerFunc(h(server.CleanAbandonedDataHandler))
+			logger.FatalIf(storageRenameDataHandler.Register(gm, server.RenameDataHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageDeleteFileHandler.Register(gm, server.DeleteFileHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageCheckPartsHandler.Register(gm, server.CheckPartsHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageReadVersionHandler.Register(gm, server.ReadVersionHandlerWS, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageWriteMetadataHandler.Register(gm, server.WriteMetadataHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageUpdateMetadataHandler.Register(gm, server.UpdateMetadataHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageDeleteVersionHandler.Register(gm, server.DeleteVersionHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageReadXLHandler.Register(gm, server.ReadXLHandlerWS, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageNSScannerHandler.RegisterNoInput(gm, server.NSScannerHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageDiskInfoHandler.Register(gm, server.DiskInfoHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(storageStatVolHandler.Register(gm, server.StatVolHandler, endpoint.Path), "unable to register handler")
+			logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerWalkDir, grid.StreamHandler{
+				Subroute:    endpoint.Path,
+				Handle:      server.WalkDirHandler,
+				OutCapacity: 1,
+			}), "unable to register handler")
 		}
 	}
 }