astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-19 20:20:25 +03:00
Code Issues Packages Projects Releases Wiki Activity

Add sufficient deadlines and countermeasures to handle hung node scenario (#19688)

Browse Source 
Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>
Signed-off-by: Harshavardhana <harsha@minio.io>
This commit is contained in:
Shubhendu
2024-05-23 04:37:14 +05:30
committed by GitHub
parent ca80eced24
commit 7c7650b7c3
34 changed files with 292 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/admin-server-info.go
+7 -5
View File
@@ -1,4 +1,4 @@
// Copyright (c) 2015-2021 MinIO, Inc.
// Copyright (c) 2015-2024 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
@@ -18,7 +18,6 @@
package cmd
import (
"context"
"math"
"net/http"
"os"
@@ -31,6 +30,7 @@ import (
"github.com/minio/madmin-go/v3"
"github.com/minio/minio/internal/config"
"github.com/minio/minio/internal/kms"
xnet "github.com/minio/pkg/v2/net"
)
// getLocalServerProperty - returns madmin.ServerProperties for only the
@@ -64,9 +64,11 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
if err := isServerResolvable(endpoint, 5*time.Second); err == nil {
network[nodeName] = string(madmin.ItemOnline)
} else {
network[nodeName] = string(madmin.ItemOffline)
// log once the error
peersLogOnceIf(context.Background(), err, nodeName)
if xnet.IsNetworkOrHostDown(err, false) {
network[nodeName] = string(madmin.ItemOffline)
} else if xnet.IsNetworkOrHostDown(err, true) {
network[nodeName] = "connection attempt timedout"
}
}
}
}
cmd/common-main.go
+2 -5
View File
@@ -404,16 +404,13 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
ctxt.FTP = ctx.StringSlice("ftp")
ctxt.SFTP = ctx.StringSlice("sftp")
ctxt.Interface = ctx.String("interface")
ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
ctxt.SendBufSize = ctx.Int("send-buf-size")
ctxt.RecvBufSize = ctx.Int("recv-buf-size")
ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
ctxt.IdleTimeout = ctx.Duration("idle-timeout")
ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
ctxt.MaxIdleConnsPerHost = ctx.Int("max-idle-conns-per-host")
ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
if conf := ctx.String("config"); len(conf) > 0 {
err = mergeServerCtxtFromConfigFile(conf, ctxt)
cmd/erasure-healing.go
+1
View File
@@ -258,6 +258,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
healTrace(healingMetricObject, startTime, bucket, object, &opts, err, &result)
}()
}
// Initialize heal result object
result = madmin.HealResultItem{
Type: madmin.HealItemObject,
cmd/erasure-multipart.go
+4 -3
View File
@@ -657,7 +657,8 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
if err != nil {
return PartInfo{}, err
}
pctx := plkctx.Context()
ctx = plkctx.Context()
defer partIDLock.Unlock(plkctx)
onlineDisks := er.getDisks()
@@ -689,7 +690,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
}
}()
erasure, err := NewErasure(pctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
if err != nil {
return pi, toObjectErr(err, bucket, object)
}
@@ -742,7 +743,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
}
}
n, err := erasure.Encode(pctx, toEncode, writers, buffer, writeQuorum)
n, err := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum)
closeBitrotWriters(writers)
if err != nil {
return pi, toObjectErr(err, bucket, object)
cmd/erasure-server-pool.go
+2 -8
View File
@@ -2472,16 +2472,10 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
for _, disk := range storageInfo.Disks {
if opts.Maintenance {
var skip bool
globalLocalDrivesMu.RLock()
for _, drive := range globalLocalDrives {
if drive != nil && drive.Endpoint().String() == disk.Endpoint {
skip = true
break
}
}
_, ok := globalLocalDrivesMap[disk.Endpoint]
globalLocalDrivesMu.RUnlock()
if skip {
if ok {
continue
}
}
cmd/globals.go
+3 -2
View File
@@ -414,8 +414,9 @@ var (
// List of local drives to this node, this is only set during server startup,
// and is only mutated by HealFormat. Hold globalLocalDrivesMu to access.
globalLocalDrives []StorageAPI
globalLocalDrivesMu sync.RWMutex
globalLocalDrives []StorageAPI
globalLocalDrivesMap = make(map[string]StorageAPI)
globalLocalDrivesMu sync.RWMutex
globalDriveMonitoring = env.Get("_MINIO_DRIVE_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn
cmd/grid.go
+5 -1
View File
@@ -36,8 +36,12 @@ var globalGridStart = make(chan struct{})
func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
hosts, local := eps.GridHosts()
lookupHost := globalDNSCache.LookupHost
g, err := grid.NewManager(ctx, grid.ManagerOptions{
Dialer: grid.ContextDialer(xhttp.DialContextWithLookupHost(globalDNSCache.LookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))),
// Pass Dialer for websocket grid, make sure we do not
// provide any DriveOPTimeout() function, as that is not
// useful over persistent connections.
Dialer: grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
Local: local,
Hosts: hosts,
AddAuth: newCachedAuthToken(),
cmd/peer-s3-client.go
+15
View File
@@ -320,6 +320,9 @@ func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts Buc
}
func (client *remotePeerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
defer cancel()
bi, err := listBucketsRPC.Call(ctx, client.gridConn(), &opts)
if err != nil {
return nil, toStorageErr(err)
@@ -345,6 +348,9 @@ func (client *remotePeerS3Client) HealBucket(ctx context.Context, bucket string,
peerS3BucketDeleted: strconv.FormatBool(opts.Remove),
})
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
defer cancel()
_, err := healBucketRPC.Call(ctx, conn, mss)
// Initialize heal result info
@@ -367,6 +373,9 @@ func (client *remotePeerS3Client) GetBucketInfo(ctx context.Context, bucket stri
peerS3BucketDeleted: strconv.FormatBool(opts.Deleted),
})
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
defer cancel()
volInfo, err := headBucketRPC.Call(ctx, conn, mss)
if err != nil {
return BucketInfo{}, toStorageErr(err)
@@ -418,6 +427,9 @@ func (client *remotePeerS3Client) MakeBucket(ctx context.Context, bucket string,
peerS3BucketForceCreate: strconv.FormatBool(opts.ForceCreate),
})
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
defer cancel()
_, err := makeBucketRPC.Call(ctx, conn, mss)
return toStorageErr(err)
}
@@ -467,6 +479,9 @@ func (client *remotePeerS3Client) DeleteBucket(ctx context.Context, bucket strin
peerS3BucketForceDelete: strconv.FormatBool(opts.Force),
})
ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout())
defer cancel()
_, err := deleteBucketRPC.Call(ctx, conn, mss)
return toStorageErr(err)
}
cmd/post-policy_test.go
+1 -1
View File
@@ -234,7 +234,7 @@ func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErr
// Call the ServeHTTP to execute the handler.
apiRouter.ServeHTTP(rec, req)
if rec.Code != test.expectedStatus {
t.Fatalf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, test.expectedStatus, rec.Code)
t.Fatalf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`, Resp: %s", i+1, instanceType, test.expectedStatus, rec.Code, rec.Body)
}
}
cmd/server-main.go
+11 -13
View File
@@ -414,10 +414,11 @@ func serverHandleCmdArgs(ctxt serverCtxt) {
setGlobalInternodeInterface(ctxt.Interface)
globalTCPOptions = xhttp.TCPOptions{
UserTimeout: int(ctxt.UserTimeout.Milliseconds()),
Interface: ctxt.Interface,
SendBufSize: ctxt.SendBufSize,
RecvBufSize: ctxt.RecvBufSize,
UserTimeout: int(ctxt.UserTimeout.Milliseconds()),
DriveOPTimeout: globalDriveConfig.GetOPTimeout,
Interface: ctxt.Interface,
SendBufSize: ctxt.SendBufSize,
RecvBufSize: ctxt.RecvBufSize,
}
// allow transport to be HTTP/1.1 for proxying.
@@ -816,6 +817,11 @@ func serverMain(ctx *cli.Context) {
}
}
var getCert certs.GetCertificateFunc
if globalTLSCerts != nil {
getCert = globalTLSCerts.GetCertificate
}
// Check for updates in non-blocking manner.
go func() {
if !globalServerCtxt.Quiet && !globalInplaceUpdateDisabled {
@@ -842,12 +848,7 @@ func serverMain(ctx *cli.Context) {
warnings = append(warnings, color.YellowBold("- Detected GOMAXPROCS(%d) < NumCPU(%d), please make sure to provide all PROCS to MinIO for optimal performance", maxProcs, cpuProcs))
}
var getCert certs.GetCertificateFunc
if globalTLSCerts != nil {
getCert = globalTLSCerts.GetCertificate
}
// Initialize gridn
// Initialize grid
bootstrapTrace("initGrid", func() {
logger.FatalIf(initGlobalGrid(GlobalContext, globalEndpoints), "Unable to configure server grid RPC services")
})
@@ -909,9 +910,6 @@ func serverMain(ctx *cli.Context) {
}
})
xhttp.SetDeploymentID(globalDeploymentID())
xhttp.SetMinIOVersion(Version)
for _, n := range globalNodes {
nodeName := n.Host
if n.IsLocal {
cmd/storage-rest-server.go
+2 -5
View File
@@ -80,11 +80,7 @@ func getStorageViaEndpoint(endpoint Endpoint) StorageAPI {
globalLocalDrivesMu.RLock()
defer globalLocalDrivesMu.RUnlock()
if len(globalLocalSetDrives) == 0 {
for _, drive := range globalLocalDrives {
if drive != nil && drive.Endpoint().Equal(endpoint) {
return drive
}
}
return globalLocalDrivesMap[endpoint.String()]
}
return globalLocalSetDrives[endpoint.PoolIdx][endpoint.SetIdx][endpoint.DiskIdx]
}
@@ -1387,6 +1383,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
defer globalLocalDrivesMu.Unlock()
globalLocalDrives = append(globalLocalDrives, storage)
globalLocalDrivesMap[endpoint.String()] = storage
globalLocalSetDrives[endpoint.PoolIdx][endpoint.SetIdx][endpoint.DiskIdx] = storage
return true
}