mirror of
https://github.com/pgsty/minio.git
synced 2026-07-19 04:00:25 +03:00
Send health diagnostics data as part of callhome (#16006)
This commit is contained in:
Shireesh Anjal
+64
-63
@@ -1911,67 +1911,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
|
||||
return ki
|
||||
}
|
||||
|
||||
// HealthInfoHandler - GET /minio/admin/v3/healthinfo
|
||||
// ----------
|
||||
// Get server health info
|
||||
func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HealthInfo")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
query := r.Form
|
||||
healthInfo := madmin.HealthInfo{
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
},
|
||||
},
|
||||
}
|
||||
healthInfoCh := make(chan madmin.HealthInfo)
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
|
||||
setCommonHeaders(w)
|
||||
|
||||
setEventStreamHeaders(w)
|
||||
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
errResp := func(err error) {
|
||||
errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
|
||||
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
healthInfo.Error = string(encodedErrorResponse)
|
||||
logger.LogIf(ctx, enc.Encode(healthInfo))
|
||||
}
|
||||
|
||||
deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
|
||||
if dstr := r.Form.Get("deadline"); dstr != "" {
|
||||
var err error
|
||||
deadline, err = time.ParseDuration(dstr)
|
||||
if err != nil {
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
|
||||
lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
|
||||
if err != nil { // returns a locked lock
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
defer nsLock.Unlock(lkctx.Cancel)
|
||||
|
||||
healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
|
||||
defer healthCancel()
|
||||
|
||||
func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) {
|
||||
hostAnonymizer := createHostAnonymizer()
|
||||
// anonAddr - Anonymizes hosts in given input string.
|
||||
anonAddr := func(addr string) string {
|
||||
@@ -2196,7 +2136,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
getAndWriteMinioConfig := func() {
|
||||
if query.Get("minioconfig") == "true" {
|
||||
config, err := readServerConfig(ctx, objectAPI)
|
||||
config, err := readServerConfig(healthCtx, objectAPI)
|
||||
if err != nil {
|
||||
healthInfo.Minio.Config = madmin.MinioConfig{
|
||||
Error: err.Error(),
|
||||
@@ -2244,7 +2184,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
getAndWriteSysConfig()
|
||||
|
||||
if query.Get("minioinfo") == "true" {
|
||||
infoMessage := getServerInfo(ctx, r)
|
||||
infoMessage := getServerInfo(healthCtx, nil)
|
||||
servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
|
||||
for _, server := range infoMessage.Servers {
|
||||
anonEndpoint := anonAddr(server.Endpoint)
|
||||
@@ -2294,6 +2234,67 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// HealthInfoHandler - GET /minio/admin/v3/healthinfo
|
||||
// ----------
|
||||
// Get server health info
|
||||
func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HealthInfo")
|
||||
defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
query := r.Form
|
||||
healthInfoCh := make(chan madmin.HealthInfo)
|
||||
enc := json.NewEncoder(w)
|
||||
|
||||
healthInfo := madmin.HealthInfo{
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
errResp := func(err error) {
|
||||
errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
|
||||
w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
|
||||
encodedErrorResponse := encodeResponse(errorResponse)
|
||||
healthInfo.Error = string(encodedErrorResponse)
|
||||
logger.LogIf(ctx, enc.Encode(healthInfo))
|
||||
}
|
||||
|
||||
deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
|
||||
if dstr := query.Get("deadline"); dstr != "" {
|
||||
var err error
|
||||
deadline, err = time.ParseDuration(dstr)
|
||||
if err != nil {
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress")
|
||||
lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline))
|
||||
if err != nil { // returns a locked lock
|
||||
errResp(err)
|
||||
return
|
||||
}
|
||||
|
||||
defer nsLock.Unlock(lkctx.Cancel)
|
||||
healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
|
||||
defer healthCancel()
|
||||
|
||||
go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo)
|
||||
|
||||
setCommonHeaders(w)
|
||||
setEventStreamHeaders(w)
|
||||
w.WriteHeader(http.StatusOK)
|
||||
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
+83
-27
@@ -18,29 +18,20 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
// callhomeSchemaVersion1 is callhome schema version 1
|
||||
callhomeSchemaVersion1 = "1"
|
||||
|
||||
// callhomeSchemaVersion is current callhome schema version.
|
||||
callhomeSchemaVersion = callhomeSchemaVersion1
|
||||
)
|
||||
|
||||
// CallhomeInfo - Contains callhome information
|
||||
type CallhomeInfo struct {
|
||||
SchemaVersion string `json:"schema_version"`
|
||||
AdminInfo madmin.InfoMessage `json:"admin_info"`
|
||||
}
|
||||
|
||||
var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
|
||||
|
||||
// initCallhome will start the callhome task in the background.
|
||||
@@ -118,26 +109,91 @@ func runCallhome(ctx context.Context, objAPI ObjectLayer) bool {
|
||||
}
|
||||
|
||||
func performCallhome(ctx context.Context) {
|
||||
err := sendCallhomeInfo(
|
||||
CallhomeInfo{
|
||||
SchemaVersion: callhomeSchemaVersion,
|
||||
AdminInfo: getServerInfo(ctx, nil),
|
||||
})
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
|
||||
deadline := 10 * time.Second // Default deadline is 10secs for callhome
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
logger.LogIf(ctx, errors.New("Callhome: object layer not ready"))
|
||||
return
|
||||
}
|
||||
|
||||
healthCtx, healthCancel := context.WithTimeout(ctx, deadline)
|
||||
defer healthCancel()
|
||||
|
||||
healthInfoCh := make(chan madmin.HealthInfo)
|
||||
|
||||
query := url.Values{}
|
||||
for _, k := range madmin.HealthDataTypesList {
|
||||
query.Set(string(k), "true")
|
||||
}
|
||||
|
||||
healthInfo := madmin.HealthInfo{
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo)
|
||||
|
||||
for {
|
||||
select {
|
||||
case hi, hasMore := <-healthInfoCh:
|
||||
if !hasMore {
|
||||
// Received all data. Send to SUBNET and return
|
||||
err := sendHealthInfo(ctx, healthInfo)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err))
|
||||
}
|
||||
return
|
||||
}
|
||||
healthInfo = hi
|
||||
case <-healthCtx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const (
|
||||
callhomeURL = "https://subnet.min.io/api/callhome"
|
||||
callhomeURLDev = "http://localhost:9000/api/callhome"
|
||||
healthURL = "https://subnet.min.io/api/health/upload"
|
||||
healthURLDev = "http://localhost:9000/api/health/upload"
|
||||
)
|
||||
|
||||
func sendCallhomeInfo(ch CallhomeInfo) error {
|
||||
url := callhomeURL
|
||||
func sendHealthInfo(ctx context.Context, healthInfo madmin.HealthInfo) error {
|
||||
url := healthURL
|
||||
if globalIsCICD {
|
||||
url = callhomeURLDev
|
||||
url = healthURLDev
|
||||
}
|
||||
_, err := globalSubnetConfig.Post(url, ch)
|
||||
|
||||
filename := fmt.Sprintf("health_%s.json.gz", UTCNow().Format("20060102150405"))
|
||||
url += "?filename=" + filename
|
||||
|
||||
_, err := globalSubnetConfig.Upload(url, filename, createHealthJSONGzip(ctx, healthInfo))
|
||||
return err
|
||||
}
|
||||
|
||||
func createHealthJSONGzip(ctx context.Context, healthInfo madmin.HealthInfo) []byte {
|
||||
var b bytes.Buffer
|
||||
gzWriter := gzip.NewWriter(&b)
|
||||
|
||||
header := struct {
|
||||
Version string `json:"version"`
|
||||
}{Version: healthInfo.Version}
|
||||
|
||||
enc := json.NewEncoder(gzWriter)
|
||||
if e := enc.Encode(header); e != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e))
|
||||
return nil
|
||||
}
|
||||
|
||||
if e := enc.Encode(healthInfo); e != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Could not encode health info: %w", e))
|
||||
return nil
|
||||
}
|
||||
|
||||
gzWriter.Flush()
|
||||
gzWriter.Close()
|
||||
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user