Add RPC tcp timeout/errs and AVG duration to prometheus (#15747)

internal/rest/rpc-stats.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package rest
+
+import (
+	"net/http"
+	"net/http/httptrace"
+	"sync/atomic"
+	"time"
+)
+
+var globalStats = struct {
+	errs uint64
+
+	tcpDialErrs     uint64
+	tcpDialCount    uint64
+	tcpDialTotalDur uint64
+}{}
+
+// RPCStats holds information about the DHCP/TCP metrics and errors
+type RPCStats struct {
+	Errs uint64
+
+	DialAvgDuration uint64
+	DialErrs        uint64
+}
+
+// GetRPCStats returns RPC stats, include calls errors and dhcp/tcp metrics
+func GetRPCStats() RPCStats {
+	s := RPCStats{
+		Errs:     atomic.LoadUint64(&globalStats.errs),
+		DialErrs: atomic.LoadUint64(&globalStats.tcpDialErrs),
+	}
+	if v := atomic.LoadUint64(&globalStats.tcpDialCount); v > 0 {
+		s.DialAvgDuration = atomic.LoadUint64(&globalStats.tcpDialTotalDur) / v
+	}
+	return s
+}
+
+// Return a function which update the global stats related to tcp connections
+func setupReqStatsUpdate(req *http.Request) (*http.Request, func()) {
+	var dialStart, dialEnd time.Time
+
+	trace := &httptrace.ClientTrace{
+		ConnectStart: func(network, addr string) {
+			dialStart = time.Now()
+		},
+		ConnectDone: func(network, addr string, err error) {
+			if err == nil {
+				dialEnd = time.Now()
+			}
+		},
+	}
+
+	return req.WithContext(httptrace.WithClientTrace(req.Context(), trace)), func() {
+		if !dialStart.IsZero() {
+			if dialEnd.IsZero() {
+				atomic.AddUint64(&globalStats.tcpDialErrs, 1)
+			} else {
+				atomic.AddUint64(&globalStats.tcpDialCount, 1)
+				atomic.AddUint64(&globalStats.tcpDialTotalDur, uint64(dialEnd.Sub(dialStart)))
+			}
+		}
+	}
+}