add optimizations to bring performance on unversioned READS (#20128)

allow non-inlined on disk to be inlined via
an unversioned ReadVersion() call, we only
need ReadXL() to resolve objects with multiple
versions only.

The choice of this block makes it to be dynamic
and chosen by the user via `mc admin config set`

Other bonus things

- Start measuring internode TTFB performance.
- Set TCP_NODELAY, TCP_CORK for low latency

internal/http/dial_linux.go

@@ -49,6 +49,11 @@ func setTCPParametersFn(opts TCPOptions) func(network, address string, c syscall
 				_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, opts.RecvBufSize)
 			}
 
+			if opts.NoDelay {
+				_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_NODELAY, 1)
+				_ = syscall.SetsockoptInt(fd, syscall.SOL_TCP, unix.TCP_CORK, 0)
+			}
+
 			// Enable TCP open
 			// https://lwn.net/Articles/508865/ - 32k queue size.
 			_ = syscall.SetsockoptInt(fd, syscall.SOL_TCP, unix.TCP_FASTOPEN, 32*1024)

internal/http/listener.go

@@ -125,6 +125,7 @@ type TCPOptions struct {
 
 	SendBufSize int              // SO_SNDBUF size for the socket connection, NOTE: this sets server and client connection
 	RecvBufSize int              // SO_RECVBUF size for the socket connection, NOTE: this sets server and client connection
+	NoDelay     bool             // Indicates callers to enable TCP_NODELAY on the net.Conn
 	Interface   string           // This is a VRF device passed via `--interface` flag
 	Trace       func(msg string) // Trace when starting.
 }
@@ -136,6 +137,7 @@ func (t TCPOptions) ForWebsocket() TCPOptions {
 		Interface:   t.Interface,
 		SendBufSize: t.SendBufSize,
 		RecvBufSize: t.RecvBufSize,
+		NoDelay:     true,
 	}
 }