add bucket level S3 received/sent bytes (#15084)

adds bucket level metrics for bytes received and sent bytes on all S3 API calls.

cmd/metrics-v2.go

@@ -293,7 +293,7 @@ func getClusterCapacityTotalBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityRawSubsystem,
 		Name:      totalBytes,
-		Help:      "Total capacity online in the cluster.",
+		Help:      "Total capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -303,7 +303,7 @@ func getClusterCapacityFreeBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityRawSubsystem,
 		Name:      freeBytes,
-		Help:      "Total free capacity online in the cluster.",
+		Help:      "Total free capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -313,7 +313,7 @@ func getClusterCapacityUsageBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityUsableSubsystem,
 		Name:      totalBytes,
-		Help:      "Total usable capacity online in the cluster.",
+		Help:      "Total usable capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -323,7 +323,7 @@ func getClusterCapacityUsageFreeBytesMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: capacityUsableSubsystem,
 		Name:      freeBytes,
-		Help:      "Total free usable capacity online in the cluster.",
+		Help:      "Total free usable capacity online in the cluster",
 		Type:      gaugeMetric,
 	}
 }
@@ -333,7 +333,7 @@ func getNodeDiskAPILatencyMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      apiLatencyMicroSec,
-		Help:      "Average last minute latency in µs for disk API storage operations.",
+		Help:      "Average last minute latency in µs for disk API storage operations",
 		Type:      gaugeMetric,
 	}
 }
@@ -343,7 +343,7 @@ func getNodeDiskUsedBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      usedBytes,
-		Help:      "Total storage used on a disk.",
+		Help:      "Total storage used on a disk",
 		Type:      gaugeMetric,
 	}
 }
@@ -353,7 +353,7 @@ func getNodeDiskFreeBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      freeBytes,
-		Help:      "Total storage available on a disk.",
+		Help:      "Total storage available on a disk",
 		Type:      gaugeMetric,
 	}
 }
@@ -363,7 +363,7 @@ func getClusterDisksOfflineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      offlineTotal,
-		Help:      "Total disks offline.",
+		Help:      "Total disks offline",
 		Type:      gaugeMetric,
 	}
 }
@@ -373,7 +373,7 @@ func getClusterDisksOnlineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      onlineTotal,
-		Help:      "Total disks online.",
+		Help:      "Total disks online",
 		Type:      gaugeMetric,
 	}
 }
@@ -383,7 +383,7 @@ func getClusterDisksTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      total,
-		Help:      "Total disks.",
+		Help:      "Total disks",
 		Type:      gaugeMetric,
 	}
 }
@@ -393,7 +393,7 @@ func getClusterDisksFreeInodes() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      freeInodes,
-		Help:      "Total free inodes.",
+		Help:      "Total free inodes",
 		Type:      gaugeMetric,
 	}
 }
@@ -403,7 +403,7 @@ func getNodeDiskTotalBytesMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: diskSubsystem,
 		Name:      totalBytes,
-		Help:      "Total storage on a disk.",
+		Help:      "Total storage on a disk",
 		Type:      gaugeMetric,
 	}
 }
@@ -428,6 +428,26 @@ func getBucketUsageQuotaTotalBytesMD() MetricDescription {
 	}
 }
 
+func getBucketTrafficReceivedBytes() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: trafficSubsystem,
+		Name:      receivedBytes,
+		Help:      "Total number of S3 bytes received for this bucket",
+		Type:      gaugeMetric,
+	}
+}
+
+func getBucketTrafficSentBytes() MetricDescription {
+	return MetricDescription{
+		Namespace: bucketMetricNamespace,
+		Subsystem: trafficSubsystem,
+		Name:      sentBytes,
+		Help:      "Total number of S3 bytes sent for this bucket",
+		Type:      gaugeMetric,
+	}
+}
+
 func getBucketUsageTotalBytesMD() MetricDescription {
 	return MetricDescription{
 		Namespace: bucketMetricNamespace,
@@ -453,7 +473,7 @@ func getBucketRepLatencyMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      latencyMilliSec,
-		Help:      "Replication latency in milliseconds.",
+		Help:      "Replication latency in milliseconds",
 		Type:      histogramMetric,
 	}
 }
@@ -463,7 +483,7 @@ func getBucketRepFailedBytesMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      failedBytes,
-		Help:      "Total number of bytes failed at least once to replicate.",
+		Help:      "Total number of bytes failed at least once to replicate",
 		Type:      gaugeMetric,
 	}
 }
@@ -473,7 +493,7 @@ func getBucketRepSentBytesMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      sentBytes,
-		Help:      "Total number of bytes replicated to the target bucket.",
+		Help:      "Total number of bytes replicated to the target bucket",
 		Type:      gaugeMetric,
 	}
 }
@@ -483,7 +503,7 @@ func getBucketRepReceivedBytesMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: replicationSubsystem,
 		Name:      receivedBytes,
-		Help:      "Total number of bytes replicated to this bucket from another source bucket.",
+		Help:      "Total number of bytes replicated to this bucket from another source bucket",
 		Type:      gaugeMetric,
 	}
 }
@@ -503,7 +523,7 @@ func getBucketObjectDistributionMD() MetricDescription {
 		Namespace: bucketMetricNamespace,
 		Subsystem: objectsSubsystem,
 		Name:      sizeDistribution,
-		Help:      "Distribution of object sizes in the bucket, includes label for the bucket name.",
+		Help:      "Distribution of object sizes in the bucket, includes label for the bucket name",
 		Type:      histogramMetric,
 	}
 }
@@ -513,7 +533,7 @@ func getInternodeFailedRequests() MetricDescription {
 		Namespace: interNodeMetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      errorsTotal,
-		Help:      "Total number of failed internode calls.",
+		Help:      "Total number of failed internode calls",
 		Type:      counterMetric,
 	}
 }
@@ -523,7 +543,7 @@ func getInterNodeSentBytesMD() MetricDescription {
 		Namespace: interNodeMetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      sentBytes,
-		Help:      "Total number of bytes sent to the other peer nodes.",
+		Help:      "Total number of bytes sent to the other peer nodes",
 		Type:      counterMetric,
 	}
 }
@@ -533,7 +553,7 @@ func getInterNodeReceivedBytesMD() MetricDescription {
 		Namespace: interNodeMetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      receivedBytes,
-		Help:      "Total number of bytes received from other peer nodes.",
+		Help:      "Total number of bytes received from other peer nodes",
 		Type:      counterMetric,
 	}
 }
@@ -553,7 +573,7 @@ func getS3ReceivedBytesMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: trafficSubsystem,
 		Name:      receivedBytes,
-		Help:      "Total number of s3 bytes received.",
+		Help:      "Total number of s3 bytes received",
 		Type:      counterMetric,
 	}
 }
@@ -643,7 +663,7 @@ func getS3RejectedAuthRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      authTotal,
-		Help:      "Total number S3 requests rejected for auth failure.",
+		Help:      "Total number S3 requests rejected for auth failure",
 		Type:      counterMetric,
 	}
 }
@@ -653,7 +673,7 @@ func getS3RejectedHeaderRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      headerTotal,
-		Help:      "Total number S3 requests rejected for invalid header.",
+		Help:      "Total number S3 requests rejected for invalid header",
 		Type:      counterMetric,
 	}
 }
@@ -663,7 +683,7 @@ func getS3RejectedTimestampRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      timestampTotal,
-		Help:      "Total number S3 requests rejected for invalid timestamp.",
+		Help:      "Total number S3 requests rejected for invalid timestamp",
 		Type:      counterMetric,
 	}
 }
@@ -673,7 +693,7 @@ func getS3RejectedInvalidRequestsTotalMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: requestsRejectedSubsystem,
 		Name:      invalidTotal,
-		Help:      "Total number S3 invalid requests.",
+		Help:      "Total number S3 invalid requests",
 		Type:      counterMetric,
 	}
 }
@@ -793,7 +813,7 @@ func getNodeOnlineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: nodesSubsystem,
 		Name:      onlineTotal,
-		Help:      "Total number of MinIO nodes online.",
+		Help:      "Total number of MinIO nodes online",
 		Type:      gaugeMetric,
 	}
 }
@@ -803,7 +823,7 @@ func getNodeOfflineTotalMD() MetricDescription {
 		Namespace: clusterMetricNamespace,
 		Subsystem: nodesSubsystem,
 		Name:      offlineTotal,
-		Help:      "Total number of MinIO nodes offline.",
+		Help:      "Total number of MinIO nodes offline",
 		Type:      gaugeMetric,
 	}
 }
@@ -823,7 +843,7 @@ func getMinIOCommitMD() MetricDescription {
 		Namespace: minioMetricNamespace,
 		Subsystem: softwareSubsystem,
 		Name:      commitInfo,
-		Help:      "Git commit hash for the MinIO release.",
+		Help:      "Git commit hash for the MinIO release",
 		Type:      gaugeMetric,
 	}
 }
@@ -833,7 +853,7 @@ func getS3TTFBDistributionMD() MetricDescription {
 		Namespace: s3MetricNamespace,
 		Subsystem: timeSubsystem,
 		Name:      ttfbDistribution,
-		Help:      "Distribution of the time to first byte across API calls.",
+		Help:      "Distribution of the time to first byte across API calls",
 		Type:      gaugeMetric,
 	}
 }
@@ -843,7 +863,7 @@ func getMinioFDOpenMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: fileDescriptorSubsystem,
 		Name:      openTotal,
-		Help:      "Total number of open file descriptors by the MinIO Server process.",
+		Help:      "Total number of open file descriptors by the MinIO Server process",
 		Type:      gaugeMetric,
 	}
 }
@@ -853,7 +873,7 @@ func getMinioFDLimitMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: fileDescriptorSubsystem,
 		Name:      limitTotal,
-		Help:      "Limit on total number of open file descriptors for the MinIO Server process.",
+		Help:      "Limit on total number of open file descriptors for the MinIO Server process",
 		Type:      gaugeMetric,
 	}
 }
@@ -923,7 +943,7 @@ func getMinIOGORoutineCountMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: goRoutines,
 		Name:      total,
-		Help:      "Total number of go routines running.",
+		Help:      "Total number of go routines running",
 		Type:      gaugeMetric,
 	}
 }
@@ -933,7 +953,7 @@ func getMinIOProcessStartTimeMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      startTime,
-		Help:      "Start time for MinIO process per node, time in seconds since Unix epoc.",
+		Help:      "Start time for MinIO process per node, time in seconds since Unix epoc",
 		Type:      gaugeMetric,
 	}
 }
@@ -943,7 +963,7 @@ func getMinIOProcessUptimeMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      upTime,
-		Help:      "Uptime for MinIO process per node in seconds.",
+		Help:      "Uptime for MinIO process per node in seconds",
 		Type:      gaugeMetric,
 	}
 }
@@ -953,7 +973,7 @@ func getMinIOProcessResidentMemory() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      memory,
-		Help:      "Resident memory size in bytes.",
+		Help:      "Resident memory size in bytes",
 		Type:      gaugeMetric,
 	}
 }
@@ -963,7 +983,7 @@ func getMinIOProcessCPUTime() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: processSubsystem,
 		Name:      cpu,
-		Help:      "Total user and system CPU time spent in seconds.",
+		Help:      "Total user and system CPU time spent in seconds",
 		Type:      counterMetric,
 	}
 }
@@ -1148,7 +1168,7 @@ func getTransitionPendingTasksMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: ilmSubsystem,
 		Name:      transitionPendingTasks,
-		Help:      "Number of pending ILM transition tasks in the queue.",
+		Help:      "Number of pending ILM transition tasks in the queue",
 		Type:      gaugeMetric,
 	}
 }
@@ -1158,7 +1178,7 @@ func getTransitionActiveTasksMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: ilmSubsystem,
 		Name:      transitionActiveTasks,
-		Help:      "Number of active ILM transition tasks.",
+		Help:      "Number of active ILM transition tasks",
 		Type:      gaugeMetric,
 	}
 }
@@ -1168,7 +1188,7 @@ func getExpiryPendingTasksMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: ilmSubsystem,
 		Name:      expiryPendingTasks,
-		Help:      "Number of pending ILM expiry tasks in the queue.",
+		Help:      "Number of pending ILM expiry tasks in the queue",
 		Type:      gaugeMetric,
 	}
 }
@@ -1210,7 +1230,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Namespace: nodeMetricNamespace,
 					Subsystem: scannerSubsystem,
 					Name:      "objects_scanned",
-					Help:      "Total number of unique objects scanned since server start.",
+					Help:      "Total number of unique objects scanned since server start",
 					Type:      counterMetric,
 				},
 				Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalObjects)),
@@ -1220,7 +1240,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Namespace: nodeMetricNamespace,
 					Subsystem: scannerSubsystem,
 					Name:      "versions_scanned",
-					Help:      "Total number of object versions scanned since server start.",
+					Help:      "Total number of object versions scanned since server start",
 					Type:      counterMetric,
 				},
 				Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalVersions)),
@@ -1230,7 +1250,7 @@ func getScannerNodeMetrics() *MetricsGroup {
 					Namespace: nodeMetricNamespace,
 					Subsystem: scannerSubsystem,
 					Name:      "directories_scanned",
-					Help:      "Total number of directories scanned since server start.",
+					Help:      "Total number of directories scanned since server start",
 					Type:      counterMetric,
 				},
 				Value: float64(atomic.LoadUint64(&globalScannerStats.accFolders)),
@@ -1674,6 +1694,24 @@ func getBucketUsageMetrics() *MetricsGroup {
 				})
 			}
 
+			recvBytes := globalBucketConnStats.getS3InputBytes(bucket)
+			if recvBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficReceivedBytes(),
+					Value:          float64(recvBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+
+			sentBytes := globalBucketConnStats.getS3OutputBytes(bucket)
+			if sentBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficSentBytes(),
+					Value:          float64(sentBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+
 			if stats.hasReplicationUsage() {
 				for arn, stat := range stats.Stats {
 					metrics = append(metrics, Metric{