perf: websocket grid connectivity for all internode communication (#18461)

This PR adds a WebSocket grid feature that allows servers to communicate via 
a single two-way connection.

There are two request types:

* Single requests, which are `[]byte => ([]byte, error)`. This is for efficient small
  roundtrips with small payloads.

* Streaming requests which are `[]byte, chan []byte => chan []byte (and error)`,
  which allows for different combinations of full two-way streams with an initial payload.

Only a single stream is created between two machines - and there is, as such, no
server/client relation since both sides can initiate and handle requests. Which server
initiates the request is decided deterministically on the server names.

Requests are made through a mux client and server, which handles message
passing, congestion, cancelation, timeouts, etc.

If a connection is lost, all requests are canceled, and the calling server will try
to reconnect. Registered handlers can operate directly on byte 
slices or use a higher-level generics abstraction.

There is no versioning of handlers/clients, and incompatible changes should
be handled by adding new handlers.

The request path can be changed to a new one for any protocol changes.

First, all servers create a "Manager." The manager must know its address 
as well as all remote addresses. This will manage all connections.
To get a connection to any remote, ask the manager to provide it given
the remote address using.

```
func (m *Manager) Connection(host string) *Connection
```

All serverside handlers must also be registered on the manager. This will
make sure that all incoming requests are served. The number of in-flight 
requests and responses must also be given for streaming requests.

The "Connection" returned manages the mux-clients. Requests issued
to the connection will be sent to the remote.

* `func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error)`
   performs a single request and returns the result. Any deadline provided on the request is
   forwarded to the server, and canceling the context will make the function return at once.

* `func (c *Connection) NewStream(ctx context.Context, h HandlerID, payload []byte) (st *Stream, err error)`
   will initiate a remote call and send the initial payload.

```Go
// A Stream is a two-way stream.
// All responses *must* be read by the caller.
// If the call is canceled through the context,
//The appropriate error will be returned.
type Stream struct {
	// Responses from the remote server.
	// Channel will be closed after an error or when the remote closes.
	// All responses *must* be read by the caller until either an error is returned or the channel is closed.
	// Canceling the context will cause the context cancellation error to be returned.
	Responses <-chan Response

	// Requests sent to the server.
	// If the handler is defined with 0 incoming capacity this will be nil.
	// Channel *must* be closed to signal the end of the stream.
	// If the request context is canceled, the stream will no longer process requests.
	Requests chan<- []byte
}

type Response struct {
	Msg []byte
	Err error
}
```

There are generic versions of the server/client handlers that allow the use of type
safe implementations for data types that support msgpack marshal/unmarshal.

internal/grid/connection_test.go

@@ -0,0 +1,209 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package grid
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/minio/minio/internal/logger/target/testlogger"
+)
+
+func TestDisconnect(t *testing.T) {
+	defer testlogger.T.SetLogTB(t)()
+	defer timeout(10 * time.Second)()
+	hosts, listeners, _ := getHosts(2)
+	dialer := &net.Dialer{
+		Timeout: 1 * time.Second,
+	}
+	errFatal := func(err error) {
+		t.Helper()
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	wrapServer := func(handler http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			t.Logf("Got a %s request for: %v", r.Method, r.URL)
+			handler.ServeHTTP(w, r)
+		})
+	}
+	connReady := make(chan struct{})
+	// We fake a local and remote server.
+	localHost := hosts[0]
+	remoteHost := hosts[1]
+	local, err := NewManager(context.Background(), ManagerOptions{
+		Dialer:       dialer.DialContext,
+		Local:        localHost,
+		Hosts:        hosts,
+		AddAuth:      func(aud string) string { return aud },
+		AuthRequest:  dummyRequestValidate,
+		BlockConnect: connReady,
+	})
+	errFatal(err)
+
+	// 1: Echo
+	errFatal(local.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) {
+		t.Log("1: server payload: ", len(payload), "bytes.")
+		return append([]byte{}, payload...), nil
+	}))
+	// 2: Return as error
+	errFatal(local.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) {
+		t.Log("2: server payload: ", len(payload), "bytes.")
+		err := RemoteErr(payload)
+		return nil, &err
+	}))
+
+	remote, err := NewManager(context.Background(), ManagerOptions{
+		Dialer:       dialer.DialContext,
+		Local:        remoteHost,
+		Hosts:        hosts,
+		AddAuth:      func(aud string) string { return aud },
+		AuthRequest:  dummyRequestValidate,
+		BlockConnect: connReady,
+	})
+	errFatal(err)
+
+	localServer := startServer(t, listeners[0], wrapServer(local.Handler()))
+	remoteServer := startServer(t, listeners[1], wrapServer(remote.Handler()))
+	close(connReady)
+
+	defer func() {
+		local.debugMsg(debugShutdown)
+		remote.debugMsg(debugShutdown)
+		remoteServer.Close()
+		localServer.Close()
+		remote.debugMsg(debugWaitForExit)
+		local.debugMsg(debugWaitForExit)
+	}()
+
+	cleanReqs := make(chan struct{})
+	gotCall := make(chan struct{})
+	defer close(cleanReqs)
+	// 1: Block forever
+	h1 := func(payload []byte) ([]byte, *RemoteErr) {
+		gotCall <- struct{}{}
+		<-cleanReqs
+		return nil, nil
+	}
+	// 2: Also block, but with streaming.
+	h2 := StreamHandler{
+		Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr {
+			gotCall <- struct{}{}
+			select {
+			case <-ctx.Done():
+				gotCall <- struct{}{}
+			case <-cleanReqs:
+				panic("should not be called")
+			}
+			return nil
+		},
+		OutCapacity: 1,
+		InCapacity:  1,
+	}
+	errFatal(remote.RegisterSingleHandler(handlerTest, h1))
+	errFatal(remote.RegisterStreamingHandler(handlerTest2, h2))
+	errFatal(local.RegisterSingleHandler(handlerTest, h1))
+	errFatal(local.RegisterStreamingHandler(handlerTest2, h2))
+
+	// local to remote
+	remoteConn := local.Connection(remoteHost)
+	errFatal(remoteConn.WaitForConnect(context.Background()))
+	const testPayload = "Hello Grid World!"
+
+	gotResp := make(chan struct{})
+	go func() {
+		start := time.Now()
+		t.Log("Roundtrip: sending request")
+		resp, err := remoteConn.Request(context.Background(), handlerTest, []byte(testPayload))
+		t.Log("Roundtrip:", time.Since(start), resp, err)
+		gotResp <- struct{}{}
+	}()
+	<-gotCall
+	remote.debugMsg(debugKillInbound)
+	local.debugMsg(debugKillInbound)
+	<-gotResp
+
+	// Must reconnect
+	errFatal(remoteConn.WaitForConnect(context.Background()))
+
+	stream, err := remoteConn.NewStream(context.Background(), handlerTest2, []byte(testPayload))
+	errFatal(err)
+	go func() {
+		for resp := range stream.responses {
+			t.Log("Resp:", resp, err)
+		}
+		gotResp <- struct{}{}
+	}()
+
+	<-gotCall
+	remote.debugMsg(debugKillOutbound)
+	local.debugMsg(debugKillOutbound)
+	errFatal(remoteConn.WaitForConnect(context.Background()))
+
+	<-gotResp
+	// Killing should cancel the context on the request.
+	<-gotCall
+}
+
+func dummyRequestValidate(r *http.Request) error {
+	return nil
+}
+
+func TestShouldConnect(t *testing.T) {
+	var c Connection
+	var cReverse Connection
+	hosts := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "x", "y", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"}
+	for x := range hosts {
+		should := 0
+		for y := range hosts {
+			if x == y {
+				continue
+			}
+			c.Local = hosts[x]
+			c.Remote = hosts[y]
+			cReverse.Local = hosts[y]
+			cReverse.Remote = hosts[x]
+			if c.shouldConnect() == cReverse.shouldConnect() {
+				t.Errorf("shouldConnect(%q, %q) != shouldConnect(%q, %q)", hosts[x], hosts[y], hosts[y], hosts[x])
+			}
+			if c.shouldConnect() {
+				should++
+			}
+		}
+		if should < 10 {
+			t.Errorf("host %q only connects to %d hosts", hosts[x], should)
+		}
+		t.Logf("host %q should connect to %d hosts", hosts[x], should)
+	}
+}
+
+func startServer(t testing.TB, listener net.Listener, handler http.Handler) (server *httptest.Server) {
+	t.Helper()
+	server = httptest.NewUnstartedServer(handler)
+	server.Config.Addr = listener.Addr().String()
+	server.Listener = listener
+	server.Start()
+	// t.Cleanup(server.Close)
+	t.Log("Started server on", server.Config.Addr, "URL:", server.URL)
+	return server
+}