Add uptime to ServiceStatus (#3690)

cmd/admin-rpc-client.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"net/url"
 	"path"
+	"sort"
 	"sync"
 	"time"
 )
@@ -39,6 +40,7 @@ type adminCmdRunner interface {
 	Restart() error
 	ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
 	ReInitDisks() error
+	Uptime() (time.Duration, error)
 }
 
 // Restart - Sends a message over channel to the go-routine
@@ -88,6 +90,28 @@ func (rc remoteAdminClient) ReInitDisks() error {
 	return rc.Call("Admin.ReInitDisks", &args, &reply)
 }
 
+// Uptime - Returns the uptime of this server. Timestamp is taken
+// after object layer is initialized.
+func (lc localAdminClient) Uptime() (time.Duration, error) {
+	if globalBootTime.IsZero() {
+		return time.Duration(0), errServerNotInitialized
+	}
+
+	return time.Now().UTC().Sub(globalBootTime), nil
+}
+
+// Uptime - returns the uptime of the server to which the RPC call is made.
+func (rc remoteAdminClient) Uptime() (time.Duration, error) {
+	args := AuthRPCArgs{}
+	reply := UptimeReply{}
+	err := rc.Call("Admin.Uptime", &args, &reply)
+	if err != nil {
+		return time.Duration(0), err
+	}
+
+	return reply.Uptime, nil
+}
+
 // adminPeer - represents an entity that implements Restart methods.
 type adminPeer struct {
 	addr      string
@@ -241,3 +265,65 @@ func reInitPeerDisks(peers adminPeers) error {
 	wg.Wait()
 	return nil
 }
+
+// uptimeSlice - used to sort uptimes in chronological order.
+type uptimeSlice []struct {
+	err    error
+	uptime time.Duration
+}
+
+func (ts uptimeSlice) Len() int {
+	return len(ts)
+}
+
+func (ts uptimeSlice) Less(i, j int) bool {
+	return ts[i].uptime < ts[j].uptime
+}
+
+func (ts uptimeSlice) Swap(i, j int) {
+	ts[i], ts[j] = ts[j], ts[i]
+}
+
+// getPeerUptimes - returns the uptime since the last time read quorum
+// was established on success. Otherwise returns errXLReadQuorum.
+func getPeerUptimes(peers adminPeers) (time.Duration, error) {
+	uptimes := make(uptimeSlice, len(peers))
+
+	// Get up time of all servers.
+	wg := sync.WaitGroup{}
+	for i, peer := range peers {
+		wg.Add(1)
+		go func(idx int, peer adminPeer) {
+			defer wg.Done()
+			uptimes[idx].uptime, uptimes[idx].err = peer.cmdRunner.Uptime()
+		}(i, peer)
+	}
+	wg.Wait()
+
+	// Sort uptimes in chronological order.
+	sort.Sort(uptimes)
+
+	// Pick the readQuorum'th uptime in chronological order. i.e,
+	// the time at which read quorum was (re-)established.
+	readQuorum := len(uptimes) / 2
+	validCount := 0
+	latestUptime := time.Duration(0)
+	for _, uptime := range uptimes {
+		if uptime.err != nil {
+			continue
+		}
+
+		validCount++
+		if validCount >= readQuorum {
+			latestUptime = uptime.uptime
+			break
+		}
+	}
+
+	// This implies there weren't read quorum number of servers up.
+	if latestUptime == time.Duration(0) {
+		return time.Duration(0), InsufficientReadQuorum{}
+	}
+
+	return latestUptime, nil
+}