mirror of
https://github.com/pgsty/minio.git
synced 2026-07-19 20:20:25 +03:00
The prometheus metrics refractoring (#8003)
The measures are consolidated to the following metrics - `disk_storage_used` : Disk space used by the disk. - `disk_storage_available`: Available disk space left on the disk. - `disk_storage_total`: Total disk space on the disk. - `disks_offline`: Total number of offline disks in current MinIO instance. - `disks_total`: Total number of disks in current MinIO instance. - `s3_requests_total`: Total number of s3 requests in current MinIO instance. - `s3_errors_total`: Total number of errors in s3 requests in current MinIO instance. - `s3_requests_current`: Total number of active s3 requests in current MinIO instance. - `internode_rx_bytes_total`: Total number of internode bytes received by current MinIO server instance. - `internode_tx_bytes_total`: Total number of bytes sent to the other nodes by current MinIO server instance. - `s3_rx_bytes_total`: Total number of s3 bytes received by current MinIO server instance. - `s3_tx_bytes_total`: Total number of s3 bytes sent by current MinIO server instance. - `minio_version_info`: Current MinIO version with commit-id. - `s3_ttfb_seconds_bucket`: Histogram that holds the latency information of the requests. And this PR also modifies the current StorageInfo queries - Decouples StorageInfo from ServerInfo . - StorageInfo is enhanced to give endpoint information. NOTE: ADMIN API VERSION IS BUMPED UP IN THIS PR Fixes #7873
This commit is contained in:
Praveen raj Mani
committed by
Harshavardhana
Harshavardhana
parent
f01d53b20f
commit
8836d57e3c
+92
-102
@@ -19,6 +19,8 @@ package cmd
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@@ -31,6 +33,8 @@ import (
|
||||
type ConnStats struct {
|
||||
totalInputBytes atomic.Uint64
|
||||
totalOutputBytes atomic.Uint64
|
||||
s3InputBytes atomic.Uint64
|
||||
s3OutputBytes atomic.Uint64
|
||||
}
|
||||
|
||||
// Increase total input bytes
|
||||
@@ -53,11 +57,33 @@ func (s *ConnStats) getTotalOutputBytes() uint64 {
|
||||
return s.totalOutputBytes.Load()
|
||||
}
|
||||
|
||||
// Return connection stats (total input/output bytes)
|
||||
// Increase outbound input bytes
|
||||
func (s *ConnStats) incS3InputBytes(n int) {
|
||||
s.s3InputBytes.Add(uint64(n))
|
||||
}
|
||||
|
||||
// Increase outbound output bytes
|
||||
func (s *ConnStats) incS3OutputBytes(n int) {
|
||||
s.s3OutputBytes.Add(uint64(n))
|
||||
}
|
||||
|
||||
// Return outbound input bytes
|
||||
func (s *ConnStats) getS3InputBytes() uint64 {
|
||||
return s.s3InputBytes.Load()
|
||||
}
|
||||
|
||||
// Return outbound output bytes
|
||||
func (s *ConnStats) getS3OutputBytes() uint64 {
|
||||
return s.s3OutputBytes.Load()
|
||||
}
|
||||
|
||||
// Return connection stats (total input/output bytes and total s3 input/output bytes)
|
||||
func (s *ConnStats) toServerConnStats() ServerConnStats {
|
||||
return ServerConnStats{
|
||||
TotalInputBytes: s.getTotalInputBytes(),
|
||||
TotalOutputBytes: s.getTotalOutputBytes(),
|
||||
S3InputBytes: s.getS3InputBytes(),
|
||||
S3OutputBytes: s.getS3OutputBytes(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -66,35 +92,55 @@ func newConnStats() *ConnStats {
|
||||
return &ConnStats{}
|
||||
}
|
||||
|
||||
// HTTPMethodStats holds statistics information about
|
||||
// a given HTTP method made by all clients
|
||||
type HTTPMethodStats struct {
|
||||
Counter atomic.Uint64
|
||||
Duration atomic.Float64
|
||||
// HTTPAPIStats holds statistics information about
|
||||
// a given API in the requests.
|
||||
type HTTPAPIStats struct {
|
||||
APIStats map[string]int
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
// Inc increments the api stats counter.
|
||||
func (stats *HTTPAPIStats) Inc(api string) {
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
if stats == nil {
|
||||
return
|
||||
}
|
||||
if stats.APIStats == nil {
|
||||
stats.APIStats = make(map[string]int)
|
||||
}
|
||||
if _, ok := stats.APIStats[api]; ok {
|
||||
stats.APIStats[api]++
|
||||
return
|
||||
}
|
||||
stats.APIStats[api] = 1
|
||||
}
|
||||
|
||||
// Dec increments the api stats counter.
|
||||
func (stats *HTTPAPIStats) Dec(api string) {
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
if stats == nil {
|
||||
return
|
||||
}
|
||||
if val, ok := stats.APIStats[api]; ok && val > 0 {
|
||||
stats.APIStats[api]--
|
||||
}
|
||||
}
|
||||
|
||||
// Load returns the recorded stats.
|
||||
func (stats *HTTPAPIStats) Load() map[string]int {
|
||||
stats.Lock()
|
||||
defer stats.Unlock()
|
||||
return stats.APIStats
|
||||
}
|
||||
|
||||
// HTTPStats holds statistics information about
|
||||
// HTTP requests made by all clients
|
||||
type HTTPStats struct {
|
||||
// HEAD request stats.
|
||||
totalHEADs HTTPMethodStats
|
||||
successHEADs HTTPMethodStats
|
||||
|
||||
// GET request stats.
|
||||
totalGETs HTTPMethodStats
|
||||
successGETs HTTPMethodStats
|
||||
|
||||
// PUT request stats.
|
||||
totalPUTs HTTPMethodStats
|
||||
successPUTs HTTPMethodStats
|
||||
|
||||
// POST request stats.
|
||||
totalPOSTs HTTPMethodStats
|
||||
successPOSTs HTTPMethodStats
|
||||
|
||||
// DELETE request stats.
|
||||
totalDELETEs HTTPMethodStats
|
||||
successDELETEs HTTPMethodStats
|
||||
currentS3Requests HTTPAPIStats
|
||||
totalS3Requests HTTPAPIStats
|
||||
totalS3Errors HTTPAPIStats
|
||||
}
|
||||
|
||||
func durationStr(totalDuration, totalCount float64) string {
|
||||
@@ -102,95 +148,39 @@ func durationStr(totalDuration, totalCount float64) string {
|
||||
}
|
||||
|
||||
// Converts http stats into struct to be sent back to the client.
|
||||
func (st HTTPStats) toServerHTTPStats() ServerHTTPStats {
|
||||
func (st *HTTPStats) toServerHTTPStats() ServerHTTPStats {
|
||||
serverStats := ServerHTTPStats{}
|
||||
serverStats.TotalHEADStats = ServerHTTPMethodStats{
|
||||
Count: st.totalHEADs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.totalHEADs.Duration.Load(), float64(st.totalHEADs.Counter.Load())),
|
||||
|
||||
serverStats.CurrentS3Requests = ServerHTTPAPIStats{
|
||||
APIStats: st.currentS3Requests.Load(),
|
||||
}
|
||||
serverStats.SuccessHEADStats = ServerHTTPMethodStats{
|
||||
Count: st.successHEADs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.successHEADs.Duration.Load(), float64(st.successHEADs.Counter.Load())),
|
||||
|
||||
serverStats.TotalS3Requests = ServerHTTPAPIStats{
|
||||
APIStats: st.totalS3Requests.Load(),
|
||||
}
|
||||
serverStats.TotalGETStats = ServerHTTPMethodStats{
|
||||
Count: st.totalGETs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.totalGETs.Duration.Load(), float64(st.totalGETs.Counter.Load())),
|
||||
}
|
||||
serverStats.SuccessGETStats = ServerHTTPMethodStats{
|
||||
Count: st.successGETs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.successGETs.Duration.Load(), float64(st.successGETs.Counter.Load())),
|
||||
}
|
||||
serverStats.TotalPUTStats = ServerHTTPMethodStats{
|
||||
Count: st.totalPUTs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.totalPUTs.Duration.Load(), float64(st.totalPUTs.Counter.Load())),
|
||||
}
|
||||
serverStats.SuccessPUTStats = ServerHTTPMethodStats{
|
||||
Count: st.successPUTs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.successPUTs.Duration.Load(), float64(st.successPUTs.Counter.Load())),
|
||||
}
|
||||
serverStats.TotalPOSTStats = ServerHTTPMethodStats{
|
||||
Count: st.totalPOSTs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.totalPOSTs.Duration.Load(), float64(st.totalPOSTs.Counter.Load())),
|
||||
}
|
||||
serverStats.SuccessPOSTStats = ServerHTTPMethodStats{
|
||||
Count: st.successPOSTs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.successPOSTs.Duration.Load(), float64(st.successPOSTs.Counter.Load())),
|
||||
}
|
||||
serverStats.TotalDELETEStats = ServerHTTPMethodStats{
|
||||
Count: st.totalDELETEs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.totalDELETEs.Duration.Load(), float64(st.totalDELETEs.Counter.Load())),
|
||||
}
|
||||
serverStats.SuccessDELETEStats = ServerHTTPMethodStats{
|
||||
Count: st.successDELETEs.Counter.Load(),
|
||||
AvgDuration: durationStr(st.successDELETEs.Duration.Load(), float64(st.successDELETEs.Counter.Load())),
|
||||
|
||||
serverStats.TotalS3Errors = ServerHTTPAPIStats{
|
||||
APIStats: st.totalS3Errors.Load(),
|
||||
}
|
||||
return serverStats
|
||||
}
|
||||
|
||||
// Update statistics from http request and response data
|
||||
func (st *HTTPStats) updateStats(r *http.Request, w *httpResponseRecorder, durationSecs float64) {
|
||||
func (st *HTTPStats) updateStats(api string, r *http.Request, w *recordAPIStats, durationSecs float64) {
|
||||
// A successful request has a 2xx response code
|
||||
successReq := (w.respStatusCode >= 200 && w.respStatusCode < 300)
|
||||
// Update stats according to method verb
|
||||
switch r.Method {
|
||||
case "HEAD":
|
||||
st.totalHEADs.Counter.Inc()
|
||||
st.totalHEADs.Duration.Add(durationSecs)
|
||||
if successReq {
|
||||
st.successHEADs.Counter.Inc()
|
||||
st.successHEADs.Duration.Add(durationSecs)
|
||||
}
|
||||
case "GET":
|
||||
st.totalGETs.Counter.Inc()
|
||||
st.totalGETs.Duration.Add(durationSecs)
|
||||
if successReq {
|
||||
st.successGETs.Counter.Inc()
|
||||
st.successGETs.Duration.Add(durationSecs)
|
||||
}
|
||||
case "PUT":
|
||||
st.totalPUTs.Counter.Inc()
|
||||
st.totalPUTs.Duration.Add(durationSecs)
|
||||
if successReq {
|
||||
st.successPUTs.Counter.Inc()
|
||||
st.totalPUTs.Duration.Add(durationSecs)
|
||||
}
|
||||
case "POST":
|
||||
st.totalPOSTs.Counter.Inc()
|
||||
st.totalPOSTs.Duration.Add(durationSecs)
|
||||
if successReq {
|
||||
st.successPOSTs.Counter.Inc()
|
||||
st.totalPOSTs.Duration.Add(durationSecs)
|
||||
}
|
||||
case "DELETE":
|
||||
st.totalDELETEs.Counter.Inc()
|
||||
st.totalDELETEs.Duration.Add(durationSecs)
|
||||
if successReq {
|
||||
st.successDELETEs.Counter.Inc()
|
||||
st.successDELETEs.Duration.Add(durationSecs)
|
||||
|
||||
if w.isS3Request && !strings.HasSuffix(r.URL.Path, prometheusMetricsPath) {
|
||||
st.totalS3Requests.Inc(api)
|
||||
if !successReq && w.respStatusCode != 0 {
|
||||
st.totalS3Errors.Inc(api)
|
||||
}
|
||||
}
|
||||
// Increment the prometheus http request response histogram with appropriate label
|
||||
httpRequestsDuration.With(prometheus.Labels{"request_type": r.Method}).Observe(durationSecs)
|
||||
|
||||
if w.isS3Request && r.Method == "GET" {
|
||||
// Increment the prometheus http request response histogram with appropriate label
|
||||
httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(durationSecs)
|
||||
}
|
||||
}
|
||||
|
||||
// Prepare new HTTPStats structure
|
||||
|
||||
Reference in New Issue
Block a user