mirror of
https://github.com/pgsty/minio.git
synced 2026-07-19 12:10:24 +03:00
Implement oboard diagnostics admin API (#9024)
- Implement a graph algorithm to test network bandwidth from every node to every other node - Saturate any network bandwidth adaptively, accounting for slow and fast network capacity - Implement parallel drive OBD tests - Implement a paging mechanism for OBD test to provide periodic updates to client - Implement Sys, Process, Host, Mem OBD Infos
This commit is contained in:
Sidhartha Mani
@@ -25,6 +25,7 @@ import (
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -38,6 +39,7 @@ import (
|
||||
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
|
||||
"github.com/minio/minio-go/v6/pkg/set"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
@@ -890,6 +892,290 @@ func (sys *NotificationSys) CollectNetPerfInfo(size int64) map[string][]ServerNe
|
||||
return reply
|
||||
}
|
||||
|
||||
// NetOBDInfo - Net OBD information
|
||||
func (sys *NotificationSys) NetOBDInfo(ctx context.Context) madmin.ServerNetOBDInfo {
|
||||
var sortedGlobalEndpoints []string
|
||||
|
||||
/*
|
||||
Ensure that only untraversed links are visited by this server
|
||||
i.e. if netOBD tests have been performed between a -> b, then do
|
||||
not run it between b -> a
|
||||
|
||||
The graph of tests looks like this
|
||||
|
||||
a b c d
|
||||
a | o | x | x | x |
|
||||
b | o | o | x | x |
|
||||
c | o | o | o | x |
|
||||
d | o | o | o | o |
|
||||
|
||||
'x's should be tested, and 'o's should be skipped
|
||||
*/
|
||||
stripPath := func(hostPath string) string {
|
||||
return strings.Split(hostPath, slashSeparator)[0]
|
||||
}
|
||||
|
||||
hostSet := set.NewStringSet()
|
||||
for _, ez := range globalEndpoints {
|
||||
for _, e := range ez.Endpoints {
|
||||
host := stripPath(e.Host)
|
||||
if hostSet.Contains(host) {
|
||||
sortedGlobalEndpoints = append(sortedGlobalEndpoints, host)
|
||||
hostSet.Add(host)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sort.Strings(sortedGlobalEndpoints)
|
||||
var remoteTargets []*peerRESTClient
|
||||
search := func(host string) *peerRESTClient {
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
if sys.peerClients[index].host.String() == host {
|
||||
return client
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
for i := 0; i < len(sortedGlobalEndpoints); i++ {
|
||||
if sortedGlobalEndpoints[i] != GetLocalPeer(globalEndpoints) {
|
||||
continue
|
||||
}
|
||||
for j := 0; j < len(sortedGlobalEndpoints); j++ {
|
||||
if j > i {
|
||||
remoteTarget := search(sortedGlobalEndpoints[j])
|
||||
if remoteTarget != nil {
|
||||
remoteTargets = append(remoteTargets, remoteTarget)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
netOBDs := make([]madmin.NetOBDInfo, len(remoteTargets))
|
||||
|
||||
for index, client := range remoteTargets {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
var err error
|
||||
netOBDs[index], err = client.NetOBDInfo(ctx)
|
||||
|
||||
addr := client.host.String()
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
netOBDs[index].Addr = addr
|
||||
if err != nil {
|
||||
netOBDs[index].Error = err.Error()
|
||||
}
|
||||
}
|
||||
return madmin.ServerNetOBDInfo{
|
||||
Net: netOBDs,
|
||||
Addr: GetLocalPeer(globalEndpoints),
|
||||
}
|
||||
}
|
||||
|
||||
// DispatchNetOBDInfo - Net OBD information from other nodes
|
||||
func (sys *NotificationSys) DispatchNetOBDInfo(ctx context.Context) []madmin.ServerNetOBDInfo {
|
||||
serverNetOBDs := []madmin.ServerNetOBDInfo{}
|
||||
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
serverNetOBD, err := sys.peerClients[index].DispatchNetOBDInfo(ctx)
|
||||
if err != nil {
|
||||
serverNetOBD.Addr = client.host.String()
|
||||
serverNetOBD.Error = err.Error()
|
||||
}
|
||||
serverNetOBDs = append(serverNetOBDs, serverNetOBD)
|
||||
}
|
||||
return serverNetOBDs
|
||||
}
|
||||
|
||||
// DriveOBDInfo - Drive OBD information
|
||||
func (sys *NotificationSys) DriveOBDInfo(ctx context.Context) []madmin.ServerDrivesOBDInfo {
|
||||
reply := make([]madmin.ServerDrivesOBDInfo, len(sys.peerClients))
|
||||
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
var err error
|
||||
reply[index], err = sys.peerClients[index].DriveOBDInfo(ctx)
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
addr := sys.peerClients[index].host.String()
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
reply[index].Addr = addr
|
||||
reply[index].Error = err.Error()
|
||||
}
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
// CPUOBDInfo - CPU OBD information
|
||||
func (sys *NotificationSys) CPUOBDInfo(ctx context.Context) []madmin.ServerCPUOBDInfo {
|
||||
reply := make([]madmin.ServerCPUOBDInfo, len(sys.peerClients))
|
||||
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
var err error
|
||||
reply[index], err = sys.peerClients[index].CPUOBDInfo(ctx)
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
addr := sys.peerClients[index].host.String()
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
reply[index].Addr = addr
|
||||
reply[index].Error = err.Error()
|
||||
}
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
// DiskHwOBDInfo - Disk HW OBD information
|
||||
func (sys *NotificationSys) DiskHwOBDInfo(ctx context.Context) []madmin.ServerDiskHwOBDInfo {
|
||||
reply := make([]madmin.ServerDiskHwOBDInfo, len(sys.peerClients))
|
||||
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
var err error
|
||||
reply[index], err = sys.peerClients[index].DiskHwOBDInfo(ctx)
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
addr := sys.peerClients[index].host.String()
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
reply[index].Addr = addr
|
||||
reply[index].Error = err.Error()
|
||||
}
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
// OsOBDInfo - Os OBD information
|
||||
func (sys *NotificationSys) OsOBDInfo(ctx context.Context) []madmin.ServerOsOBDInfo {
|
||||
reply := make([]madmin.ServerOsOBDInfo, len(sys.peerClients))
|
||||
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
var err error
|
||||
reply[index], err = sys.peerClients[index].OsOBDInfo(ctx)
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
addr := sys.peerClients[index].host.String()
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
reply[index].Addr = addr
|
||||
reply[index].Error = err.Error()
|
||||
}
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
// MemOBDInfo - Mem OBD information
|
||||
func (sys *NotificationSys) MemOBDInfo(ctx context.Context) []madmin.ServerMemOBDInfo {
|
||||
reply := make([]madmin.ServerMemOBDInfo, len(sys.peerClients))
|
||||
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
var err error
|
||||
reply[index], err = sys.peerClients[index].MemOBDInfo(ctx)
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
addr := sys.peerClients[index].host.String()
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
reply[index].Addr = addr
|
||||
reply[index].Error = err.Error()
|
||||
}
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
// ProcOBDInfo - Process OBD information
|
||||
func (sys *NotificationSys) ProcOBDInfo(ctx context.Context) []madmin.ServerProcOBDInfo {
|
||||
reply := make([]madmin.ServerProcOBDInfo, len(sys.peerClients))
|
||||
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
var err error
|
||||
reply[index], err = sys.peerClients[index].ProcOBDInfo(ctx)
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
addr := sys.peerClients[index].host.String()
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
reply[index].Addr = addr
|
||||
reply[index].Error = err.Error()
|
||||
}
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
// DrivePerfInfo - Drive speed (read and write) information
|
||||
func (sys *NotificationSys) DrivePerfInfo(size int64) []madmin.ServerDrivesPerfInfo {
|
||||
reply := make([]madmin.ServerDrivesPerfInfo, len(sys.peerClients))
|
||||
|
||||
Reference in New Issue
Block a user