mirror of
https://github.com/pgsty/minio.git
synced 2026-07-23 06:00:24 +03:00
Add support for resource metrics (#18057)
Add a new endpoint for "resource" metrics `/v2/metrics/resource` This should return system metrics related to drives, network, CPU and memory. Except for drives, other metrics should have corresponding "avg" and "max" values also. Reuse the real-time feature to capture the required data, introducing CPU and memory metrics in it. Collect the data every minute and keep updating the average and max values accordingly, returning the latest values when the API is called.
This commit is contained in:
Shireesh Anjal
+73
-110
@@ -132,6 +132,9 @@ const (
|
||||
capacityRawSubsystem MetricSubsystem = "capacity_raw"
|
||||
capacityUsableSubsystem MetricSubsystem = "capacity_usable"
|
||||
driveSubsystem MetricSubsystem = "drive"
|
||||
interfaceSubsystem MetricSubsystem = "if"
|
||||
memSubsystem MetricSubsystem = "mem"
|
||||
cpuSubsystem MetricSubsystem = "cpu_avg"
|
||||
storageClassSubsystem MetricSubsystem = "storage_class"
|
||||
fileDescriptorSubsystem MetricSubsystem = "file_descriptor"
|
||||
goRoutines MetricSubsystem = "go_routine"
|
||||
@@ -539,12 +542,12 @@ func getNodeRRSParityMD() MetricDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func getNodeDrivesFreeInodes() MetricDescription {
|
||||
func getNodeDrivesFreeInodesMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: nodeMetricNamespace,
|
||||
Subsystem: driveSubsystem,
|
||||
Name: freeInodes,
|
||||
Help: "Total free inodes",
|
||||
Help: "Free inodes on a drive",
|
||||
Type: gaugeMetric,
|
||||
}
|
||||
}
|
||||
@@ -3207,7 +3210,7 @@ func getLocalStorageMetrics() *MetricsGroup {
|
||||
})
|
||||
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getNodeDrivesFreeInodes(),
|
||||
Description: getNodeDrivesFreeInodesMD(),
|
||||
Value: float64(disk.FreeInodes),
|
||||
VariableLabels: map[string]string{"drive": disk.DrivePath},
|
||||
})
|
||||
@@ -3546,6 +3549,61 @@ func getKMSMetrics() *MetricsGroup {
|
||||
return mg
|
||||
}
|
||||
|
||||
func collectMetric(metric Metric, labels []string, values []string, metricName string, out chan<- prometheus.Metric) {
|
||||
if metric.Description.Type == histogramMetric {
|
||||
if metric.Histogram == nil {
|
||||
return
|
||||
}
|
||||
for k, v := range metric.Histogram {
|
||||
pmetric, err := prometheus.NewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(string(metric.Description.Namespace),
|
||||
string(metric.Description.Subsystem),
|
||||
string(metric.Description.Name)),
|
||||
metric.Description.Help,
|
||||
append(labels, metric.HistogramBucketLabel),
|
||||
metric.StaticLabels,
|
||||
),
|
||||
prometheus.GaugeValue,
|
||||
float64(v),
|
||||
append(values, k)...)
|
||||
if err != nil {
|
||||
// Enable for debugging
|
||||
if serverDebugLog {
|
||||
logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), metricName+"-metrics-histogram")
|
||||
}
|
||||
} else {
|
||||
out <- pmetric
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
metricType := prometheus.GaugeValue
|
||||
if metric.Description.Type == counterMetric {
|
||||
metricType = prometheus.CounterValue
|
||||
}
|
||||
pmetric, err := prometheus.NewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(string(metric.Description.Namespace),
|
||||
string(metric.Description.Subsystem),
|
||||
string(metric.Description.Name)),
|
||||
metric.Description.Help,
|
||||
labels,
|
||||
metric.StaticLabels,
|
||||
),
|
||||
metricType,
|
||||
metric.Value,
|
||||
values...)
|
||||
if err != nil {
|
||||
// Enable for debugging
|
||||
if serverDebugLog {
|
||||
logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), metricName+"-metrics")
|
||||
}
|
||||
} else {
|
||||
out <- pmetric
|
||||
}
|
||||
}
|
||||
|
||||
type minioBucketCollector struct {
|
||||
metricsGroups []*MetricsGroup
|
||||
desc *prometheus.Desc
|
||||
@@ -3570,52 +3628,7 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
|
||||
defer wg.Done()
|
||||
for metric := range in {
|
||||
labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
|
||||
if metric.Description.Type == histogramMetric {
|
||||
if metric.Histogram == nil {
|
||||
continue
|
||||
}
|
||||
for k, v := range metric.Histogram {
|
||||
pmetric, err := prometheus.NewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(string(metric.Description.Namespace),
|
||||
string(metric.Description.Subsystem),
|
||||
string(metric.Description.Name)),
|
||||
metric.Description.Help,
|
||||
append(labels, metric.HistogramBucketLabel),
|
||||
metric.StaticLabels,
|
||||
),
|
||||
prometheus.GaugeValue,
|
||||
float64(v),
|
||||
append(values, k)...)
|
||||
if err != nil {
|
||||
logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), "bucket-metrics-histogram")
|
||||
} else {
|
||||
out <- pmetric
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
metricType := prometheus.GaugeValue
|
||||
if metric.Description.Type == counterMetric {
|
||||
metricType = prometheus.CounterValue
|
||||
}
|
||||
pmetric, err := prometheus.NewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(string(metric.Description.Namespace),
|
||||
string(metric.Description.Subsystem),
|
||||
string(metric.Description.Name)),
|
||||
metric.Description.Help,
|
||||
labels,
|
||||
metric.StaticLabels,
|
||||
),
|
||||
metricType,
|
||||
metric.Value,
|
||||
values...)
|
||||
if err != nil {
|
||||
logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "bucket-metrics")
|
||||
} else {
|
||||
out <- pmetric
|
||||
}
|
||||
collectMetric(metric, labels, values, "bucket", out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3650,59 +3663,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
|
||||
defer wg.Done()
|
||||
for metric := range in {
|
||||
labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
|
||||
if metric.Description.Type == histogramMetric {
|
||||
if metric.Histogram == nil {
|
||||
continue
|
||||
}
|
||||
for k, v := range metric.Histogram {
|
||||
pmetric, err := prometheus.NewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(string(metric.Description.Namespace),
|
||||
string(metric.Description.Subsystem),
|
||||
string(metric.Description.Name)),
|
||||
metric.Description.Help,
|
||||
append(labels, metric.HistogramBucketLabel),
|
||||
metric.StaticLabels,
|
||||
),
|
||||
prometheus.GaugeValue,
|
||||
float64(v),
|
||||
append(values, k)...)
|
||||
|
||||
if err != nil {
|
||||
// Enable for debugging
|
||||
if serverDebugLog {
|
||||
logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v:%v", err, values, metric.Histogram), "cluster-metrics-histogram")
|
||||
}
|
||||
} else {
|
||||
out <- pmetric
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
metricType := prometheus.GaugeValue
|
||||
if metric.Description.Type == counterMetric {
|
||||
metricType = prometheus.CounterValue
|
||||
}
|
||||
pmetric, err := prometheus.NewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(string(metric.Description.Namespace),
|
||||
string(metric.Description.Subsystem),
|
||||
string(metric.Description.Name)),
|
||||
metric.Description.Help,
|
||||
labels,
|
||||
metric.StaticLabels,
|
||||
),
|
||||
metricType,
|
||||
metric.Value,
|
||||
values...)
|
||||
if err != nil {
|
||||
// Enable for debugging
|
||||
if serverDebugLog {
|
||||
logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "cluster-metrics")
|
||||
}
|
||||
} else {
|
||||
out <- pmetric
|
||||
}
|
||||
collectMetric(metric, labels, values, "cluster", out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3835,11 +3796,11 @@ func newMinioCollectorNode(metricsGroups []*MetricsGroup) *minioNodeCollector {
|
||||
}
|
||||
}
|
||||
|
||||
func metricsBucketHandler() http.Handler {
|
||||
func metricsHTTPHandler(c prometheus.Collector, funcName string) http.Handler {
|
||||
registry := prometheus.NewRegistry()
|
||||
|
||||
// Report all other metrics
|
||||
logger.CriticalIf(GlobalContext, registry.Register(bucketCollector))
|
||||
logger.CriticalIf(GlobalContext, registry.Register(c))
|
||||
|
||||
// DefaultGatherers include golang metrics and process metrics.
|
||||
gatherers := prometheus.Gatherers{
|
||||
@@ -3849,16 +3810,14 @@ func metricsBucketHandler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
|
||||
if ok {
|
||||
tc.FuncName = "handler.MetricsBucket"
|
||||
tc.FuncName = funcName
|
||||
tc.ResponseRecorder.LogErrBody = true
|
||||
}
|
||||
|
||||
mfs, err := gatherers.Gather()
|
||||
if err != nil {
|
||||
if len(mfs) == 0 {
|
||||
writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
|
||||
return
|
||||
}
|
||||
if err != nil && len(mfs) == 0 {
|
||||
writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
contentType := expfmt.Negotiate(r.Header)
|
||||
@@ -3878,6 +3837,10 @@ func metricsBucketHandler() http.Handler {
|
||||
})
|
||||
}
|
||||
|
||||
func metricsBucketHandler() http.Handler {
|
||||
return metricsHTTPHandler(bucketCollector, "handler.MetricsBucket")
|
||||
}
|
||||
|
||||
func metricsServerHandler() http.Handler {
|
||||
registry := prometheus.NewRegistry()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user