Add sufficient deadlines and countermeasures to handle hung node scenario (#19688)

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>

internal/http/dial_linux.go

@@ -26,6 +26,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/minio/minio/internal/deadlineconn"
 	"golang.org/x/sys/unix"
 )
 
@@ -39,8 +40,8 @@ func setTCPParametersFn(opts TCPOptions) func(network, address string, c syscall
 
 			_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
 
+			// Enable custom socket send/recv buffers.
 			if opts.SendBufSize > 0 {
-				// Enable big buffers
 				_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_SNDBUF, opts.SendBufSize)
 			}
 
@@ -84,7 +85,9 @@ func setTCPParametersFn(opts TCPOptions) func(network, address string, c syscall
 			//    https://blog.cloudflare.com/when-tcp-sockets-refuse-to-die/
 			// This is a sensitive configuration, it is better to set it to high values, > 60 secs since it can
 			// affect clients reading data with a very slow pace  (disappropriate with socket buffer sizes)
-			_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, opts.UserTimeout)
+			if opts.UserTimeout > 0 {
+				_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, opts.UserTimeout)
+			}
 
 			if opts.Interface != "" {
 				if h, _, err := net.SplitHostPort(address); err == nil {
@@ -111,6 +114,16 @@ func NewInternodeDialContext(dialTimeout time.Duration, opts TCPOptions) DialCon
 			Timeout: dialTimeout,
 			Control: setTCPParametersFn(opts),
 		}
-		return dialer.DialContext(ctx, network, addr)
+		conn, err := dialer.DialContext(ctx, network, addr)
+		if err != nil {
+			return nil, err
+		}
+		if opts.DriveOPTimeout != nil {
+			// Read deadlines are sufficient for now as per various
+			// scenarios of hung node detection, we may add Write deadlines
+			// if needed later on.
+			return deadlineconn.New(conn).WithReadDeadline(opts.DriveOPTimeout()), nil
+		}
+		return conn, nil
 	}
 }

internal/http/listener.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"net"
 	"syscall"
+	"time"
 )
 
 type acceptResult struct {
@@ -117,13 +118,27 @@ func (listener *httpListener) Addrs() (addrs []net.Addr) {
 
 // TCPOptions specify customizable TCP optimizations on raw socket
 type TCPOptions struct {
-	UserTimeout int              // this value is expected to be in milliseconds
+	UserTimeout int // this value is expected to be in milliseconds
+
+	// When the net.Conn is a remote drive this value is honored, we close the connection to remote peer proactively.
+	DriveOPTimeout func() time.Duration
+
 	SendBufSize int              // SO_SNDBUF size for the socket connection, NOTE: this sets server and client connection
 	RecvBufSize int              // SO_RECVBUF size for the socket connection, NOTE: this sets server and client connection
 	Interface   string           // This is a VRF device passed via `--interface` flag
 	Trace       func(msg string) // Trace when starting.
 }
 
+// ForWebsocket returns TCPOptions valid for websocket net.Conn
+func (t TCPOptions) ForWebsocket() TCPOptions {
+	return TCPOptions{
+		UserTimeout: t.UserTimeout,
+		Interface:   t.Interface,
+		SendBufSize: t.SendBufSize,
+		RecvBufSize: t.RecvBufSize,
+	}
+}
+
 // newHTTPListener - creates new httpListener object which is interface compatible to net.Listener.
 // httpListener is capable to
 // * listen to multiple addresses