Print storage errors during distributed initialization (#6441)

This commit will print connection failures to other disks in other nodes
after 5 retries. It is useful for users to understand why the
distribued cluster fails to boot up.

cmd/storage-rpc-client.go

@@ -104,6 +104,8 @@ func toStorageErr(err error) error {
 type StorageRPCClient struct {
 	*RPCClient
 	connected bool
+	// Plain error of the last RPC call
+	lastRPCError error
 }
 
 // Stringer provides a canonicalized representation of network device.
@@ -114,6 +116,11 @@ func (client *StorageRPCClient) String() string {
 	return url.String()
 }
 
+// LastError - returns the last RPC call result, nil or error if any
+func (client *StorageRPCClient) LastError() error {
+	return client.lastRPCError
+}
+
 // Close - closes underneath RPC client.
 func (client *StorageRPCClient) Close() error {
 	client.connected = false
@@ -125,14 +132,22 @@ func (client *StorageRPCClient) IsOnline() bool {
 	return client.connected
 }
 
+func (client *StorageRPCClient) connect() {
+	err := client.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{})
+	client.lastRPCError = err
+	client.connected = err == nil
+}
+
 func (client *StorageRPCClient) call(handler string, args interface {
 	SetAuthArgs(args AuthArgs)
 }, reply interface{}) error {
+
 	if !client.connected {
 		return errDiskNotFound
 	}
 
 	err := client.Call(handler, args, reply)
+	client.lastRPCError = err
 	if err == nil {
 		return nil
 	}
@@ -318,6 +333,7 @@ func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
 	logger.FatalIf(err, "Unable to parse storage RPC Host", context.Background())
 	rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
 	logger.FatalIf(err, "Unable to initialize storage RPC client", context.Background())
-	rpcClient.connected = rpcClient.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{}) == nil
+	// Attempt first try connection and save error if any.
+	rpcClient.connect()
 	return rpcClient
 }