Improve tracing & notification scalability (#18903)

* Perform JSON encoding on remote machines and only forward byte slices. * Migrate tracing & notification to WebSockets.
2026-07-20 04:30:26 +03:00 · 2024-01-30 12:49:02 -08:00
parent 80ca120088
commit 6da4a9c7bb
11 changed files with 451 additions and 213 deletions
@@ -22,19 +22,20 @@ import (
 	"context"
 	"encoding/gob"
 	"encoding/hex"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"net/url"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"

 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/bandwidth"
-	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/grid"
 	xhttp "github.com/minio/minio/internal/http"
-	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
 	"github.com/minio/pkg/v2/logger/message/log"
@@ -46,6 +47,66 @@ import (
 type peerRESTClient struct {
 	host       *xnet.Host
 	restClient *rest.Client
+	gridHost   string
+	// Function that returns the grid connection for this peer when initialized.
+	// Will return nil if the grid connection is not initialized yet.
+	gridConn func() *grid.Connection
+}
+
+// Returns a peer rest client.
+func newPeerRESTClient(peer *xnet.Host, gridHost string) *peerRESTClient {
+	scheme := "http"
+	if globalIsTLS {
+		scheme = "https"
+	}
+
+	serverURL := &url.URL{
+		Scheme: scheme,
+		Host:   peer.String(),
+		Path:   peerRESTPath,
+	}
+
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
+	// Use a separate client to avoid recursive calls.
+	healthClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
+	healthClient.NoMetrics = true
+
+	// Construct a new health function.
+	restClient.HealthCheckFn = func() bool {
+		ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout)
+		defer cancel()
+		respBody, err := healthClient.Call(ctx, peerRESTMethodHealth, nil, nil, -1)
+		xhttp.DrainBody(respBody)
+		return !isNetworkError(err)
+	}
+	var gridConn atomic.Pointer[grid.Connection]
+
+	return &peerRESTClient{
+		host: peer, restClient: restClient, gridHost: gridHost,
+		gridConn: func() *grid.Connection {
+			// Lazy initialization of grid connection.
+			// When we create this peer client, the grid connection is likely not yet initialized.
+			if gridHost == "" {
+				logger.LogOnceIf(context.Background(), fmt.Errorf("gridHost is empty for peer %s", peer.String()), peer.String()+":gridHost")
+				return nil
+			}
+			gc := gridConn.Load()
+			if gc != nil {
+				return gc
+			}
+			gm := globalGrid.Load()
+			if gm == nil {
+				return nil
+			}
+			gc = gm.Connection(gridHost)
+			if gc == nil {
+				logger.LogOnceIf(context.Background(), fmt.Errorf("gridHost %q not found for peer %s", gridHost, peer.String()), peer.String()+":gridHost")
+				return nil
+			}
+			gridConn.Store(gc)
+			return gc
+		},
+	}
 }

 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
@@ -254,7 +315,7 @@ func (client *peerRESTClient) GetResourceMetrics(ctx context.Context) (<-chan Me
 	go func(ch chan<- Metric) {
 		defer func() {
 			xhttp.DrainBody(respBody)
-			xioutil.SafeClose(ch)
+			close(ch)
 		}()
 		for {
 			var metric Metric
@@ -618,92 +679,61 @@ func (client *peerRESTClient) LoadTransitionTierConfig(ctx context.Context) erro
 	return nil
 }

-func (client *peerRESTClient) doTrace(traceCh chan<- madmin.TraceInfo, doneCh <-chan struct{}, traceOpts madmin.ServiceTraceOpts) {
-	values := make(url.Values)
-	traceOpts.AddParams(values)
-
-	// To cancel the REST request in case doneCh gets closed.
-	ctx, cancel := context.WithCancel(GlobalContext)
-
-	cancelCh := make(chan struct{})
-	defer xioutil.SafeClose(cancelCh)
-	go func() {
-		select {
-		case <-doneCh:
-		case <-cancelCh:
-			// There was an error in the REST request.
-		}
-		cancel()
-	}()
-
-	respBody, err := client.callWithContext(ctx, peerRESTMethodTrace, values, nil, -1)
-	defer xhttp.DrainBody(respBody)
-
-	if err != nil {
+func (client *peerRESTClient) doTrace(ctx context.Context, traceCh chan<- []byte, traceOpts madmin.ServiceTraceOpts) {
+	gridConn := client.gridConn()
+	if gridConn == nil {
 		return
 	}

-	dec := gob.NewDecoder(respBody)
-	for {
-		var info madmin.TraceInfo
-		if err = dec.Decode(&info); err != nil {
-			return
-		}
-		if len(info.NodeName) > 0 {
-			select {
-			case traceCh <- info:
-			default:
-				// Do not block on slow receivers.
-			}
-		}
+	payload, err := json.Marshal(traceOpts)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
 	}
+
+	st, err := gridConn.NewStream(ctx, grid.HandlerTrace, payload)
+	if err != nil {
+		return
+	}
+	st.Results(func(b []byte) error {
+		select {
+		case traceCh <- b:
+		default:
+			// Do not block on slow receivers.
+			// Just recycle the buffer.
+			grid.PutByteBuffer(b)
+		}
+		return nil
+	})
 }

-func (client *peerRESTClient) doListen(listenCh chan<- event.Event, doneCh <-chan struct{}, v url.Values) {
-	// To cancel the REST request in case doneCh gets closed.
-	ctx, cancel := context.WithCancel(GlobalContext)
-
-	cancelCh := make(chan struct{})
-	defer xioutil.SafeClose(cancelCh)
-	go func() {
-		select {
-		case <-doneCh:
-		case <-cancelCh:
-			// There was an error in the REST request.
-		}
-		cancel()
-	}()
-
-	respBody, err := client.callWithContext(ctx, peerRESTMethodListen, v, nil, -1)
-	defer xhttp.DrainBody(respBody)
-
+func (client *peerRESTClient) doListen(ctx context.Context, listenCh chan<- []byte, v url.Values) {
+	conn := client.gridConn()
+	if conn == nil {
+		return
+	}
+	st, err := listenHandler.Call(ctx, conn, grid.NewURLValuesWith(v))
 	if err != nil {
 		return
 	}
-
-	dec := gob.NewDecoder(respBody)
-	for {
-		var ev event.Event
-		if err := dec.Decode(&ev); err != nil {
-			return
+	st.Results(func(b *grid.Bytes) error {
+		select {
+		case listenCh <- *b:
+		default:
+			// Do not block on slow receivers.
+			b.Recycle()
 		}
-		if len(ev.EventVersion) > 0 {
-			select {
-			case listenCh <- ev:
-			default:
-				// Do not block on slow receivers.
-			}
-		}
-	}
+		return nil
+	})
 }

 // Listen - listen on peers.
-func (client *peerRESTClient) Listen(listenCh chan<- event.Event, doneCh <-chan struct{}, v url.Values) {
+func (client *peerRESTClient) Listen(ctx context.Context, listenCh chan<- []byte, v url.Values) {
 	go func() {
 		for {
-			client.doListen(listenCh, doneCh, v)
+			client.doListen(ctx, listenCh, v)
 			select {
-			case <-doneCh:
+			case <-ctx.Done():
 				return
 			default:
 				// There was error in the REST request, retry after sometime as probably the peer is down.
@@ -714,12 +744,13 @@ func (client *peerRESTClient) Listen(listenCh chan<- event.Event, doneCh <-chan
 }

 // Trace - send http trace request to peer nodes
-func (client *peerRESTClient) Trace(traceCh chan<- madmin.TraceInfo, doneCh <-chan struct{}, traceOpts madmin.ServiceTraceOpts) {
+func (client *peerRESTClient) Trace(ctx context.Context, traceCh chan<- []byte, traceOpts madmin.ServiceTraceOpts) {
 	go func() {
 		for {
-			client.doTrace(traceCh, doneCh, traceOpts)
+			// Blocks until context is canceled or an error occurs.
+			client.doTrace(ctx, traceCh, traceOpts)
 			select {
-			case <-doneCh:
+			case <-ctx.Done():
 				return
 			default:
 				// There was error in the REST request, retry after sometime as probably the peer is down.
@@ -734,7 +765,7 @@ func (client *peerRESTClient) doConsoleLog(logCh chan log.Info, doneCh <-chan st
 	ctx, cancel := context.WithCancel(GlobalContext)

 	cancelCh := make(chan struct{})
-	defer xioutil.SafeClose(cancelCh)
+	defer close(cancelCh)
 	go func() {
 		select {
 		case <-doneCh:
@@ -789,6 +820,7 @@ func newPeerRestClients(endpoints EndpointServerPools) (remote, all []*peerRESTC
 		// Only useful in distributed setups
 		return nil, nil
 	}
+
 	hosts := endpoints.hostsSorted()
 	remote = make([]*peerRESTClient, 0, len(hosts))
 	all = make([]*peerRESTClient, len(hosts))
@@ -796,7 +828,7 @@ func newPeerRestClients(endpoints EndpointServerPools) (remote, all []*peerRESTC
 		if host == nil {
 			continue
 		}
-		all[i] = newPeerRESTClient(host)
+		all[i] = newPeerRESTClient(host, endpoints.FindGridHostsFromPeer(host))
 		remote = append(remote, all[i])
 	}
 	if len(all) != len(remote)+1 {
@@ -805,36 +837,6 @@ func newPeerRestClients(endpoints EndpointServerPools) (remote, all []*peerRESTC
 	return remote, all
 }

-// Returns a peer rest client.
-func newPeerRESTClient(peer *xnet.Host) *peerRESTClient {
-	scheme := "http"
-	if globalIsTLS {
-		scheme = "https"
-	}
-
-	serverURL := &url.URL{
-		Scheme: scheme,
-		Host:   peer.String(),
-		Path:   peerRESTPath,
-	}
-
-	restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
-	// Use a separate client to avoid recursive calls.
-	healthClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
-	healthClient.NoMetrics = true
-
-	// Construct a new health function.
-	restClient.HealthCheckFn = func() bool {
-		ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout)
-		defer cancel()
-		respBody, err := healthClient.Call(ctx, peerRESTMethodHealth, nil, nil, -1)
-		xhttp.DrainBody(respBody)
-		return !isNetworkError(err)
-	}
-
-	return &peerRESTClient{host: peer, restClient: restClient}
-}
-
 // MonitorBandwidth - send http trace request to peer nodes
 func (client *peerRESTClient) MonitorBandwidth(ctx context.Context, buckets []string) (*bandwidth.BucketBandwidthReport, error) {
 	values := make(url.Values)
@@ -861,7 +863,7 @@ func (client *peerRESTClient) GetPeerMetrics(ctx context.Context) (<-chan Metric
 	go func(ch chan<- Metric) {
 		defer func() {
 			xhttp.DrainBody(respBody)
-			xioutil.SafeClose(ch)
+			close(ch)
 		}()
 		for {
 			var metric Metric
@@ -888,7 +890,7 @@ func (client *peerRESTClient) GetPeerBucketMetrics(ctx context.Context) (<-chan
 	go func(ch chan<- Metric) {
 		defer func() {
 			xhttp.DrainBody(respBody)
-			xioutil.SafeClose(ch)
+			close(ch)
 		}()
 		for {
 			var metric Metric
@@ -1026,7 +1028,7 @@ func (client *peerRESTClient) GetReplicationMRF(ctx context.Context, bucket stri
 	go func(ch chan madmin.ReplicationMRF) {
 		defer func() {
 			xhttp.DrainBody(respBody)
-			xioutil.SafeClose(ch)
+			close(ch)
 		}()
 		for {
 			var entry madmin.ReplicationMRF