Add more bootstrap info in debug mode (#17362)

internal/http/listener.go

@@ -120,8 +120,9 @@ func (listener *httpListener) Addrs() (addrs []net.Addr) {
 
 // TCPOptions specify customizable TCP optimizations on raw socket
 type TCPOptions struct {
-	UserTimeout int    // this value is expected to be in milliseconds
-	Interface   string // this is a VRF device passed via `--interface` flag
+	UserTimeout int              // this value is expected to be in milliseconds
+	Interface   string           // this is a VRF device passed via `--interface` flag
+	Trace       func(msg string) // Trace when starting.
 }
 
 // newHTTPListener - creates new httpListener object which is interface compatible to net.Listener.
@@ -162,6 +163,10 @@ func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions)
 	for _, serverAddr := range serverAddrs {
 		var l net.Listener
 		if l, err = listenCfg.Listen(ctx, "tcp", serverAddr); err != nil {
+			if opts.Trace != nil {
+				opts.Trace(fmt.Sprint("listenCfg.Listen: ", err.Error()))
+			}
+
 			if isLocalhost && strings.HasPrefix(serverAddr, "[::1") {
 				continue
 			}
@@ -171,12 +176,18 @@ func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions)
 		tcpListener, ok := l.(*net.TCPListener)
 		if !ok {
 			err = fmt.Errorf("unexpected listener type found %v, expected net.TCPListener", l)
+			if opts.Trace != nil {
+				opts.Trace(fmt.Sprint("net.TCPListener: ", err.Error()))
+			}
+
 			if isLocalhost && strings.HasPrefix(serverAddr, "[::1") {
 				continue
 			}
 			return nil, err
 		}
-
+		if opts.Trace != nil {
+			opts.Trace(fmt.Sprint("adding listener to ", tcpListener.Addr()))
+		}
 		tcpListeners = append(tcpListeners, tcpListener)
 	}
 
@@ -196,6 +207,9 @@ func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions)
 		acceptCh:     make(chan acceptResult, len(tcpListeners)),
 	}
 	listener.ctx, listener.ctxCanceler = context.WithCancel(ctx)
+	if opts.Trace != nil {
+		opts.Trace(fmt.Sprint("opening ", len(listener.tcpListeners), " listeners"))
+	}
 	listener.start()
 
 	return listener, nil

internal/rest/client.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"net/http/httputil"
 	"net/url"
 	"path"
 	"strings"
@@ -89,6 +90,9 @@ type Client struct {
 	// Avoid metrics update if set to true
 	NoMetrics bool
 
+	// TraceOutput will print debug information on non-200 calls if set.
+	TraceOutput io.Writer // Debug trace output
+
 	httpClient   *http.Client
 	url          *url.URL
 	newAuthToken func(audience string) string
@@ -222,6 +226,66 @@ func (r *respBodyMonitor) errorStatus(err error) {
 	}
 }
 
+// dumpHTTP - dump HTTP request and response.
+func (c *Client) dumpHTTP(req *http.Request, resp *http.Response) {
+	// Starts http dump.
+	_, err := fmt.Fprintln(c.TraceOutput, "---------START-HTTP---------")
+	if err != nil {
+		return
+	}
+
+	// Filter out Signature field from Authorization header.
+	origAuth := req.Header.Get("Authorization")
+	if origAuth != "" {
+		req.Header.Set("Authorization", "**REDACTED**")
+	}
+
+	// Only display request header.
+	reqTrace, err := httputil.DumpRequestOut(req, false)
+	if err != nil {
+		return
+	}
+
+	// Write request to trace output.
+	_, err = fmt.Fprint(c.TraceOutput, string(reqTrace))
+	if err != nil {
+		return
+	}
+
+	// Only display response header.
+	var respTrace []byte
+
+	// For errors we make sure to dump response body as well.
+	if resp.StatusCode != http.StatusOK &&
+		resp.StatusCode != http.StatusPartialContent &&
+		resp.StatusCode != http.StatusNoContent {
+		respTrace, err = httputil.DumpResponse(resp, true)
+		if err != nil {
+			return
+		}
+	} else {
+		respTrace, err = httputil.DumpResponse(resp, false)
+		if err != nil {
+			return
+		}
+	}
+
+	// Write response to trace output.
+	_, err = fmt.Fprint(c.TraceOutput, strings.TrimSuffix(string(respTrace), "\r\n"))
+	if err != nil {
+		return
+	}
+
+	// Ends the http dump.
+	_, err = fmt.Fprintln(c.TraceOutput, "---------END-HTTP---------")
+	if err != nil {
+		return
+	}
+
+	// Returns success.
+	return
+}
+
 // Call - make a REST call with context.
 func (c *Client) Call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
 	urlStr := c.url.String()
@@ -263,6 +327,12 @@ func (c *Client) Call(ctx context.Context, method string, values url.Values, bod
 		return nil, &NetworkError{err}
 	}
 
+	// If trace is enabled, dump http request and response,
+	// except when the traceErrorsOnly enabled and the response's status code is ok
+	if c.TraceOutput != nil && resp.StatusCode != http.StatusOK {
+		c.dumpHTTP(req, resp)
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		// If server returns 412 pre-condition failed, it would
 		// mean that authentication succeeded, but another