Add version as part of Server: header (#7100)

This was agreed after discussing with @abperiasamy, we
borrowed the idea from Apache's own documentation.

README.md

@@ -53,6 +53,15 @@ chmod +x minio
 ./minio server /data
 ```
 
+| Platform| Architecture | URL|
+| ----------| -------- | ------|
+|GNU/Linux|ppc64le|https://dl.minio.io/server/minio/release/linux-ppc64le/minio |
+```sh
+wget https://dl.minio.io/server/minio/release/linux-ppc64le/minio
+chmod +x minio
+./minio server /data
+```
+
 ## Microsoft Windows
 ### Binary Download
 | Platform| Architecture | URL|

cmd/admin-handlers.go

@@ -17,31 +17,32 @@
 package cmd
 
 import (
-	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"os"
 	"strconv"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/tidwall/gjson"
+	"github.com/tidwall/sjson"
+
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
+	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/mem"
+	xnet "github.com/minio/minio/pkg/net"
 	"github.com/minio/minio/pkg/quick"
-	"github.com/tidwall/gjson"
-	"github.com/tidwall/sjson"
 )
 
 const (
@@ -95,6 +96,11 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServiceStatus")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -107,13 +113,8 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 		CommitID: CommitID,
 	}
 
-	// Fetch uptimes from all peers. This may fail to due to lack
-	// of read-quorum availability.
-	uptime, err := getPeerUptimes(globalAdminPeers)
-	if err != nil {
-		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
-		return
-	}
+	// Fetch uptimes from all peers and pick the latest.
+	uptime := getPeerUptimes(globalNotificationSys.ServerInfo(ctx))
 
 	// Create API response
 	serverStatus := madmin.ServiceStatus{
@@ -127,6 +128,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
 	// Reply with storage information (across nodes in a
 	// distributed setup) as json.
 	writeSuccessResponseJSON(w, jsonBytes)
@@ -140,6 +142,11 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
 func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServiceStopNRestart")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -149,7 +156,6 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	var sa madmin.ServiceAction
 	err := json.NewDecoder(r.Body).Decode(&sa)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
 		return
 	}
@@ -169,7 +175,15 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
 	// Reply to the client before restarting minio server.
 	writeSuccessResponseHeadersOnly(w)
 
-	sendServiceCmd(globalAdminPeers, serviceSig)
+	// Notify all other Minio peers signal service.
+	for _, nerr := range globalNotificationSys.SignalService(serviceSig) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	globalServiceSignalCh <- serviceSig
 }
 
 // ServerProperties holds some server information such as, version, region
@@ -233,6 +247,12 @@ type ServerInfo struct {
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ServerInfo")
 
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	// Authenticate request
 
 	// Setting the region as empty so as the mc server info command is irrespective to the region.
@@ -242,39 +262,33 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Web service response
-	reply := make([]ServerInfo, len(globalAdminPeers))
-
-	var wg sync.WaitGroup
-
-	// Gather server information for all nodes
-	for i, p := range globalAdminPeers {
-		wg.Add(1)
-
-		// Gather information from a peer in a goroutine
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-
-			// Initialize server info at index
-			reply[idx] = ServerInfo{Addr: peer.addr}
-
-			serverInfoData, err := peer.cmdRunner.ServerInfo()
-			if err != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr)
-				ctx := logger.SetReqInfo(ctx, reqInfo)
-				logger.LogIf(ctx, err)
-				reply[idx].Error = err.Error()
-				return
-			}
-
-			reply[idx].Data = &serverInfoData
-		}(i, p)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+		return
 	}
 
-	wg.Wait()
+	serverInfo := globalNotificationSys.ServerInfo(ctx)
+	// Once we have received all the ServerInfo from peers
+	// add the local peer server info as well.
+	serverInfo = append(serverInfo, ServerInfo{
+		Addr: thisAddr.String(),
+		Data: &ServerInfoData{
+			StorageInfo: objectAPI.StorageInfo(ctx),
+			ConnStats:   globalConnStats.toServerConnStats(),
+			HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+			Properties: ServerProperties{
+				Uptime:   UTCNow().Sub(globalBootTime),
+				Version:  Version,
+				CommitID: CommitID,
+				SQSARN:   globalNotificationSys.GetARNList(),
+				Region:   globalServerConfig.GetRegion(),
+			},
+		},
+	})
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(reply)
+	jsonBytes, err := json.Marshal(serverInfo)
 	if err != nil {
 		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
@@ -294,6 +308,24 @@ type ServerDrivesPerfInfo struct {
 	Perf  []disk.Performance `json:"perf"`
 }
 
+// ServerCPULoadInfo holds informantion about cpu utilization
+// of one minio node. It also reports any errors if encountered
+// while trying to reach this server.
+type ServerCPULoadInfo struct {
+	Addr  string     `json:"addr"`
+	Error string     `json:"error,omitempty"`
+	Load  []cpu.Load `json:"load"`
+}
+
+// ServerMemUsageInfo holds informantion about memory utilization
+// of one minio node. It also reports any errors if encountered
+// while trying to reach this server.
+type ServerMemUsageInfo struct {
+	Addr  string      `json:"addr"`
+	Error string      `json:"error,omitempty"`
+	Usage []mem.Usage `json:"usage"`
+}
+
 // PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
 // ----------
 // Get all performance information based on input type
@@ -301,6 +333,13 @@ type ServerDrivesPerfInfo struct {
 func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "PerfInfo")
 
+	// Get object layer instance.
+	objLayer := newObjectLayerFn()
+	if objLayer == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	// Authenticate request
 	// Setting the region as empty so as the mc server info command is irrespective to the region.
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
@@ -313,8 +352,14 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 	perfType := vars["perfType"]
 
 	if perfType == "drive" {
+		info := objLayer.StorageInfo(ctx)
+		if !(info.Backend.Type == BackendFS || info.Backend.Type == BackendErasure) {
+
+			writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+			return
+		}
 		// Get drive performance details from local server's drive(s)
-		dp := localEndpointsPerf(globalEndpoints)
+		dp := localEndpointsDrivePerf(globalEndpoints)
 
 		// Notify all other Minio peers to report drive performance numbers
 		dps := globalNotificationSys.DrivePerfInfo()
@@ -330,8 +375,42 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 		// Reply with performance information (across nodes in a
 		// distributed setup) as json.
 		writeSuccessResponseJSON(w, jsonBytes)
+	} else if perfType == "cpu" {
+		// Get CPU load details from local server's cpu(s)
+		cpu := localEndpointsCPULoad(globalEndpoints)
+		// Notify all other Minio peers to report cpu load numbers
+		cpus := globalNotificationSys.CPULoadInfo()
+		cpus = append(cpus, cpu)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(cpus)
+		if err != nil {
+			writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with cpu load information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
+	} else if perfType == "mem" {
+		// Get mem usage details from local server(s)
+		m := localEndpointsMemUsage(globalEndpoints)
+		// Notify all other Minio peers to report mem usage numbers
+		mems := globalNotificationSys.MemUsageInfo()
+		mems = append(mems, m)
+
+		// Marshal API response
+		jsonBytes, err := json.Marshal(mems)
+		if err != nil {
+			writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+			return
+		}
+
+		// Reply with mem usage information (across nodes in a
+		// distributed setup) as json.
+		writeSuccessResponseJSON(w, jsonBytes)
 	} else {
-		writeErrorResponseJSON(w, ErrNotImplemented, r.URL)
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
 	}
 	return
 }
@@ -350,6 +429,11 @@ type StartProfilingResult struct {
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "StartProfiling")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
@@ -359,31 +443,53 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	vars := mux.Vars(r)
 	profiler := vars["profilerType"]
 
-	startProfilingResult := make([]StartProfilingResult, len(globalAdminPeers))
-
-	// Call StartProfiling function on all nodes and save results
-	wg := sync.WaitGroup{}
-	for i, peer := range globalAdminPeers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			result := StartProfilingResult{NodeName: peer.addr}
-			if err := peer.cmdRunner.StartProfiling(profiler); err != nil {
-				result.Error = err.Error()
-				return
-			}
-			result.Success = true
-			startProfilingResult[idx] = result
-		}(i, peer)
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+		return
+	}
+
+	// Start profiling on remote servers.
+	hostErrs := globalNotificationSys.StartProfiling(profiler)
+
+	// Start profiling locally as well.
+	{
+		if globalProfiler != nil {
+			globalProfiler.Stop()
+		}
+		prof, err := startProfiler(profiler, "")
+		if err != nil {
+			hostErrs = append(hostErrs, NotificationPeerErr{
+				Host: *thisAddr,
+				Err:  err,
+			})
+		} else {
+			globalProfiler = prof
+			hostErrs = append(hostErrs, NotificationPeerErr{
+				Host: *thisAddr,
+			})
+		}
+	}
+
+	var startProfilingResult []StartProfilingResult
+
+	for _, nerr := range hostErrs {
+		result := StartProfilingResult{NodeName: nerr.Host.String()}
+		if nerr.Err != nil {
+			result.Error = nerr.Err.Error()
+		} else {
+			result.Success = true
+		}
+		startProfilingResult = append(startProfilingResult, result)
 	}
-	wg.Wait()
 
 	// Create JSON result and send it to the client
 	startProfilingResultInBytes, err := json.Marshal(startProfilingResult)
 	if err != nil {
-		writeCustomErrorResponseJSON(w, http.StatusInternalServerError, err.Error(), r.URL)
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
 		return
 	}
+
 	writeSuccessResponseJSON(w, []byte(startProfilingResultInBytes))
 }
 
@@ -411,51 +517,18 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(w, adminAPIErr, r.URL)
 		return
 	}
 
-	profilingDataFound := false
-
-	// Initialize a zip writer which will provide a zipped content
-	// of profiling data of all nodes
-	zipWriter := zip.NewWriter(w)
-	defer zipWriter.Close()
-
-	for i, peer := range globalAdminPeers {
-		// Get profiling data from a node
-		data, err := peer.cmdRunner.DownloadProfilingData()
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
-			continue
-		}
-
-		profilingDataFound = true
-
-		// Send profiling data to zip as file
-		header, err := zip.FileInfoHeader(dummyFileInfo{
-			name:    fmt.Sprintf("profiling-%d", i),
-			size:    int64(len(data)),
-			mode:    0600,
-			modTime: time.Now().UTC(),
-			isDir:   false,
-			sys:     nil,
-		})
-		if err != nil {
-			continue
-		}
-		writer, err := zipWriter.CreateHeader(header)
-		if err != nil {
-			continue
-		}
-		if _, err = io.Copy(writer, bytes.NewBuffer(data)); err != nil {
-			return
-		}
-	}
-
-	if !profilingDataFound {
+	if !globalNotificationSys.DownloadProfilingData(ctx, w) {
 		writeErrorResponseJSON(w, ErrAdminProfilerNotEnabled, r.URL)
 		return
 	}
@@ -885,6 +958,11 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetUserStatus")
 
+	if globalNotificationSys == nil || globalIAMSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
@@ -921,10 +999,10 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -935,7 +1013,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -989,10 +1067,10 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1003,7 +1081,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1035,7 +1113,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1062,10 +1140,10 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1076,7 +1154,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1127,10 +1205,10 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1141,7 +1219,7 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalIAMSys == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1174,10 +1252,10 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
 	}
 
 	// Notify all other Minio peers to reload users
-	for host, err := range globalNotificationSys.LoadUsers() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 }
@@ -1188,7 +1266,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1405,7 +1483,7 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
 		return
 	}
@@ -1468,10 +1546,10 @@ func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
 	}
 
 	// Notify all other Minio peers to update credentials
-	for host, err := range globalNotificationSys.LoadCredentials() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadCredentials() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 

cmd/admin-handlers_test.go

@@ -123,7 +123,8 @@ var (
         "username": "",
         "password": "",
         "reconnectInterval": 0,
-        "keepAliveInterval": 0
+	"keepAliveInterval": 0,
+	"queueDir": ""
       }
     },
     "mysql": {
@@ -533,7 +534,6 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	var wg sync.WaitGroup
 
@@ -607,7 +607,6 @@ func TestServiceSetCreds(t *testing.T) {
 	// single node setup, this degenerates to a simple function
 	// call under the hood.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	credentials := globalServerConfig.GetCredential()
 
@@ -706,7 +705,6 @@ func TestGetConfigHandler(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for get-config mgmt REST API.
 	queryVal := url.Values{}
@@ -735,7 +733,6 @@ func TestSetConfigHandler(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}
@@ -807,7 +804,6 @@ func TestAdminServerInfo(t *testing.T) {
 
 	// Initialize admin peers to make admin RPC calls.
 	globalMinioAddr = "127.0.0.1:9000"
-	initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
 
 	// Prepare query params for set-config mgmt REST API.
 	queryVal := url.Values{}

cmd/admin-heal-ops.go

@@ -641,10 +641,10 @@ func (h *healSequence) healDiskFormat() error {
 	// Healing succeeded notify the peers to reload format and re-initialize disks.
 	// We will not notify peers only if healing succeeded.
 	if err == nil {
-		for host, rerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
-			if rerr != nil {
-				logger.GetReqInfo(h.ctx).SetTags("peerAddress", host.String())
-				logger.LogIf(h.ctx, rerr)
+		for _, nerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(h.ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(h.ctx, nerr.Err)
 			}
 		}
 	}

cmd/admin-router.go

@@ -63,9 +63,9 @@ func registerAdminRouter(router *mux.Router, enableIAM bool) {
 
 		/// Health operations
 
-		// Performance command - return performance details based on input type
-		adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
 	}
+	// Performance command - return performance details based on input type
+	adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
 
 	// Profiling operations
 	adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).

cmd/admin-rpc-client.go

@@ -1,261 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2014, 2015, 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"crypto/tls"
-	"fmt"
-	"net"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/minio/minio/cmd/logger"
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
-
-// AdminRPCClient - admin RPC client talks to admin RPC server.
-type AdminRPCClient struct {
-	*RPCClient
-}
-
-// SignalService - calls SignalService RPC.
-func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error) {
-	args := SignalServiceArgs{Sig: signal}
-	reply := VoidReply{}
-
-	return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
-}
-
-// ServerInfo - returns the server info of the server to which the RPC call is made.
-func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
-	err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
-	return sid, err
-}
-
-// StartProfiling - starts profiling in the remote server.
-func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
-	args := StartProfilingArgs{Profiler: profiler}
-	reply := VoidReply{}
-	return rpcClient.Call(adminServiceName+".StartProfiling", &args, &reply)
-}
-
-// DownloadProfilingData - returns profiling data of the remote server.
-func (rpcClient *AdminRPCClient) DownloadProfilingData() ([]byte, error) {
-	args := AuthArgs{}
-	var reply []byte
-
-	err := rpcClient.Call(adminServiceName+".DownloadProfilingData", &args, &reply)
-	return reply, err
-}
-
-// NewAdminRPCClient - returns new admin RPC client.
-func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
-	scheme := "http"
-	if globalIsSSL {
-		scheme = "https"
-	}
-
-	serviceURL := &xnet.URL{
-		Scheme: scheme,
-		Host:   host.String(),
-		Path:   adminServicePath,
-	}
-
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: host.Name,
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	rpcClient, err := NewRPCClient(
-		RPCClientArgs{
-			NewAuthTokenFunc: newAuthToken,
-			RPCVersion:       globalRPCAPIVersion,
-			ServiceName:      adminServiceName,
-			ServiceURL:       serviceURL,
-			TLSConfig:        tlsConfig,
-		},
-	)
-	if err != nil {
-		return nil, err
-	}
-
-	return &AdminRPCClient{rpcClient}, nil
-}
-
-// adminCmdRunner - abstracts local and remote execution of admin
-// commands like service stop and service restart.
-type adminCmdRunner interface {
-	SignalService(s serviceSignal) error
-	ServerInfo() (ServerInfoData, error)
-	StartProfiling(string) error
-	DownloadProfilingData() ([]byte, error)
-}
-
-// adminPeer - represents an entity that implements admin API RPCs.
-type adminPeer struct {
-	addr      string
-	cmdRunner adminCmdRunner
-	isLocal   bool
-}
-
-// type alias for a collection of adminPeer.
-type adminPeers []adminPeer
-
-// makeAdminPeers - helper function to construct a collection of adminPeer.
-func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
-	localAddr := GetLocalPeer(endpoints)
-	if strings.HasPrefix(localAddr, "127.0.0.1:") {
-		// Use first IPv4 instead of loopback address.
-		localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
-	}
-	if strings.HasPrefix(localAddr, "[::1]:") {
-		// Use first IPv4 instead of loopback address.
-		localAddr = net.JoinHostPort(localIP6.ToSlice()[0], globalMinioPort)
-	}
-
-	adminPeerList = append(adminPeerList, adminPeer{
-		addr:      localAddr,
-		cmdRunner: localAdminClient{},
-		isLocal:   true,
-	})
-
-	for _, hostStr := range GetRemotePeers(endpoints) {
-		host, err := xnet.ParseHost(hostStr)
-		logger.FatalIf(err, "Unable to parse Admin RPC Host")
-		rpcClient, err := NewAdminRPCClient(host)
-		logger.FatalIf(err, "Unable to initialize Admin RPC Client")
-		adminPeerList = append(adminPeerList, adminPeer{
-			addr:      hostStr,
-			cmdRunner: rpcClient,
-		})
-	}
-
-	return adminPeerList
-}
-
-// Initialize global adminPeer collection.
-func initGlobalAdminPeers(endpoints EndpointList) {
-	globalAdminPeers = makeAdminPeers(endpoints)
-}
-
-// invokeServiceCmd - Invoke Restart/Stop command.
-func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
-	switch cmd {
-	case serviceRestart, serviceStop:
-		err = cp.cmdRunner.SignalService(cmd)
-	}
-	return err
-}
-
-// sendServiceCmd - Invoke Restart command on remote peers
-// adminPeer followed by on the local peer.
-func sendServiceCmd(cps adminPeers, cmd serviceSignal) {
-	// Send service command like stop or restart to all remote nodes and finally run on local node.
-	errs := make([]error, len(cps))
-	var wg sync.WaitGroup
-	remotePeers := cps[1:]
-	for i := range remotePeers {
-		wg.Add(1)
-		go func(idx int) {
-			defer wg.Done()
-			// we use idx+1 because remotePeers slice is 1 position shifted w.r.t cps
-			errs[idx+1] = invokeServiceCmd(remotePeers[idx], cmd)
-		}(i)
-	}
-	wg.Wait()
-	errs[0] = invokeServiceCmd(cps[0], cmd)
-}
-
-// uptimeSlice - used to sort uptimes in chronological order.
-type uptimeSlice []struct {
-	err    error
-	uptime time.Duration
-}
-
-func (ts uptimeSlice) Len() int {
-	return len(ts)
-}
-
-func (ts uptimeSlice) Less(i, j int) bool {
-	return ts[i].uptime < ts[j].uptime
-}
-
-func (ts uptimeSlice) Swap(i, j int) {
-	ts[i], ts[j] = ts[j], ts[i]
-}
-
-// getPeerUptimes - returns the uptime since the last time read quorum
-// was established on success. Otherwise returns errXLReadQuorum.
-func getPeerUptimes(peers adminPeers) (time.Duration, error) {
-	// In a single node Erasure or FS backend setup the uptime of
-	// the setup is the uptime of the single minio server
-	// instance.
-	if !globalIsDistXL {
-		return UTCNow().Sub(globalBootTime), nil
-	}
-
-	uptimes := make(uptimeSlice, len(peers))
-
-	// Get up time of all servers.
-	wg := sync.WaitGroup{}
-	for i, peer := range peers {
-		wg.Add(1)
-		go func(idx int, peer adminPeer) {
-			defer wg.Done()
-			serverInfoData, rpcErr := peer.cmdRunner.ServerInfo()
-			uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
-		}(i, peer)
-	}
-	wg.Wait()
-
-	// Sort uptimes in chronological order.
-	sort.Sort(uptimes)
-
-	// Pick the readQuorum'th uptime in chronological order. i.e,
-	// the time at which read quorum was (re-)established.
-	readQuorum := len(uptimes) / 2
-	validCount := 0
-	latestUptime := time.Duration(0)
-	for _, uptime := range uptimes {
-		if uptime.err != nil {
-			logger.LogIf(context.Background(), uptime.err)
-			continue
-		}
-
-		validCount++
-		if validCount >= readQuorum {
-			latestUptime = uptime.uptime
-			break
-		}
-	}
-
-	// Less than readQuorum "Admin.Uptime" RPC call returned
-	// successfully, so read-quorum unavailable.
-	if validCount < readQuorum {
-		return time.Duration(0), InsufficientReadQuorum{}
-	}
-
-	return latestUptime, nil
-}

cmd/admin-rpc-server.go

@@ -1,86 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"path"
-
-	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/logger"
-	xrpc "github.com/minio/minio/cmd/rpc"
-)
-
-const adminServiceName = "Admin"
-const adminServiceSubPath = "/admin"
-
-var adminServicePath = path.Join(minioReservedBucketPath, adminServiceSubPath)
-
-// adminRPCReceiver - Admin RPC receiver for admin RPC server.
-type adminRPCReceiver struct {
-	local *localAdminClient
-}
-
-// SignalServiceArgs - provides the signal argument to SignalService RPC
-type SignalServiceArgs struct {
-	AuthArgs
-	Sig serviceSignal
-}
-
-// SignalService - Send a restart or stop signal to the service
-func (receiver *adminRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
-	return receiver.local.SignalService(args.Sig)
-}
-
-// ServerInfo - returns the server info when object layer was initialized on this server.
-func (receiver *adminRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) (err error) {
-	*reply, err = receiver.local.ServerInfo()
-	return err
-}
-
-// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
-type StartProfilingArgs struct {
-	AuthArgs
-	Profiler string
-}
-
-// StartProfiling - starts profiling of this server
-func (receiver *adminRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
-	return receiver.local.StartProfiling(args.Profiler)
-}
-
-// DownloadProfilingData - stops and returns profiling data of this server
-func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) (err error) {
-	*reply, err = receiver.local.DownloadProfilingData()
-	return
-}
-
-// NewAdminRPCServer - returns new admin RPC server.
-func NewAdminRPCServer() (*xrpc.Server, error) {
-	rpcServer := xrpc.NewServer()
-	if err := rpcServer.RegisterName(adminServiceName, &adminRPCReceiver{&localAdminClient{}}); err != nil {
-		return nil, err
-	}
-	return rpcServer, nil
-}
-
-// registerAdminRPCRouter - creates and registers Admin RPC server and its router.
-func registerAdminRPCRouter(router *mux.Router) {
-	rpcServer, err := NewAdminRPCServer()
-	logger.FatalIf(err, "Unable to initialize Lock RPC Server")
-	subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
-	subrouter.Path(adminServiceSubPath).HandlerFunc(httpTraceHdrs(rpcServer.ServeHTTP))
-}

cmd/admin-rpc_test.go

@@ -1,161 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"net/http"
-	"net/http/httptest"
-	"testing"
-	"time"
-
-	xnet "github.com/minio/minio/pkg/net"
-)
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// localAdminClient and AdminRPCClient are adminCmdRunner interface compatible,
-// hence below test functions are available for both clients.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-///////////////////////////////////////////////////////////////////////////////
-//
-// Admin RPC server, adminRPCReceiver and AdminRPCClient are
-// inter-dependent, below test functions are sufficient to test all of them.
-//
-///////////////////////////////////////////////////////////////////////////////
-
-func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
-	defer func(sigChan chan serviceSignal) { globalServiceSignalCh = sigChan }(globalServiceSignalCh)
-	globalServiceSignalCh = make(chan serviceSignal, 10)
-
-	testCases := []struct {
-		signal    serviceSignal
-		expectErr bool
-	}{
-		{serviceRestart, false},
-		{serviceStop, false},
-		{serviceStatus, true},
-		{serviceSignal(100), true},
-	}
-
-	for i, testCase := range testCases {
-		err := client.SignalService(testCase.signal)
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
-	tmpGlobalBootTime := globalBootTime
-	tmpGlobalObjectAPI := globalObjectAPI
-	tmpGlobalConnStats := globalConnStats
-	tmpGlobalHTTPStats := globalHTTPStats
-	tmpGlobalNotificationSys := globalNotificationSys
-	defer func() {
-		globalBootTime = tmpGlobalBootTime
-		globalObjectAPI = tmpGlobalObjectAPI
-		globalConnStats = tmpGlobalConnStats
-		globalHTTPStats = tmpGlobalHTTPStats
-		globalNotificationSys = tmpGlobalNotificationSys
-	}()
-
-	endpoints := new(EndpointList)
-
-	notificationSys := NewNotificationSys(globalServerConfig, *endpoints)
-
-	testCases := []struct {
-		bootTime        time.Time
-		objectAPI       ObjectLayer
-		connStats       *ConnStats
-		httpStats       *HTTPStats
-		notificationSys *NotificationSys
-		expectErr       bool
-	}{
-		{UTCNow(), &DummyObjectLayer{}, newConnStats(), newHTTPStats(), notificationSys, false},
-		{time.Time{}, nil, nil, nil, nil, true},
-		{UTCNow(), nil, nil, nil, nil, true},
-	}
-
-	for i, testCase := range testCases {
-		globalBootTime = testCase.bootTime
-		globalObjectAPI = testCase.objectAPI
-		globalConnStats = testCase.connStats
-		globalHTTPStats = testCase.httpStats
-		globalNotificationSys = testCase.notificationSys
-		_, err := client.ServerInfo()
-		expectErr := (err != nil)
-
-		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
-		}
-	}
-}
-
-func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
-	rpcServer, err := NewAdminRPCServer()
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	httpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		rpcServer.ServeHTTP(w, r)
-	}))
-
-	url, err := xnet.ParseURL(httpServer.URL)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	host, err := xnet.ParseHost(url.Host)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	prevGlobalServerConfig := globalServerConfig
-	globalServerConfig = newServerConfig()
-
-	rpcClient, err := NewAdminRPCClient(host)
-	if err != nil {
-		t.Fatalf("unexpected error %v", err)
-	}
-
-	return httpServer, rpcClient, prevGlobalServerConfig
-}
-
-func TestAdminRPCClientSignalService(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerSignalService(t, rpcClient)
-}
-
-func TestAdminRPCClientServerInfo(t *testing.T) {
-	httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
-	defer httpServer.Close()
-	defer func() {
-		globalServerConfig = prevGlobalServerConfig
-	}()
-
-	testAdminCmdRunnerServerInfo(t, rpcClient)
-}

cmd/api-headers.go

@@ -36,7 +36,7 @@ func mustGetRequestID(t time.Time) string {
 
 // Write http common headers
 func setCommonHeaders(w http.ResponseWriter) {
-	w.Header().Set("Server", globalServerUserAgent)
+	w.Header().Set("Server", "Minio/"+ReleaseTag)
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
 	if region := globalServerConfig.GetRegion(); region != "" {

cmd/common-main.go

@@ -78,18 +78,23 @@ func loadLoggers() {
 
 }
 
-func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) *ConfigDir {
+func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
 	var dir string
+	var dirSet bool
 
 	switch {
 	case ctx.IsSet(option):
 		dir = ctx.String(option)
+		dirSet = true
 	case ctx.GlobalIsSet(option):
 		dir = ctx.GlobalString(option)
+		dirSet = true
 		// cli package does not expose parent's option option.  Below code is workaround.
 		if dir == "" || dir == getDefaultDir() {
+			dirSet = false // Unset to false since GlobalIsSet() true is a false positive.
 			if ctx.Parent().GlobalIsSet(option) {
 				dir = ctx.Parent().GlobalString(option)
+				dirSet = true
 			}
 		}
 	default:
@@ -111,7 +116,7 @@ func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() s
 
 	logger.FatalIf(mkdirAllIgnorePerm(dirAbs), "Unable to create directory specified %s=%s", option, dir)
 
-	return &ConfigDir{path: dirAbs}
+	return &ConfigDir{path: dirAbs}, dirSet
 }
 
 func handleCommonCmdArgs(ctx *cli.Context) {
@@ -142,8 +147,17 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	}
 
 	// Set all config, certs and CAs directories.
-	globalConfigDir = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
-	globalCertsDir = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get)
+	var configSet, certsSet bool
+	globalConfigDir, configSet = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
+	globalCertsDir, certsSet = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get)
+
+	// Remove this code when we deprecate and remove config-dir.
+	// This code is to make sure we inherit from the config-dir
+	// option if certs-dir is not provided.
+	if !certsSet && configSet {
+		globalCertsDir = &ConfigDir{path: filepath.Join(globalConfigDir.Get(), certsDir)}
+	}
+
 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
 
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())

cmd/config-current_test.go

@@ -233,7 +233,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": "", "queueDir": ""}}}}`, false},
 
 		// Test 28 - Test NSQ
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},

cmd/config-migrate.go

@@ -2415,6 +2415,14 @@ func migrateV27ToV28() error {
 
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
+	// Construct path to config.json for the given bucket.
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
+	// Verify if config was already available in .minio.sys in which case, nothing more to be done.
+	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
+		return err
+	}
+
 	defer func() {
 		// Rename config.json to config.json.deprecated only upon
 		// success of this function.
@@ -2423,8 +2431,6 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 		}
 	}()
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	// Construct path to config.json for the given bucket.
 	transactionConfigFile := configFile + ".transaction"
 
 	// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
@@ -2436,7 +2442,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	}
 	defer objLock.Unlock()
 
-	// Verify if backend already has the file.
+	// Verify if backend already has the file (after holding lock)
 	if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
 		return err
 	} // if errConfigNotFound proceed to migrate..
@@ -2459,6 +2465,15 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
+	// Check if the config version is latest, if not migrate.
+	ok, _, err := checkConfigVersion(objAPI, configFile, serverConfigVersion)
+	if err != nil {
+		return err
+	}
+	if ok {
+		return nil
+	}
+
 	// Construct path to config.json for the given bucket.
 	transactionConfigFile := configFile + ".transaction"
 

cmd/config-versions.go

@@ -893,7 +893,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS.
+// serverConfigV33 is just like version '32', removes clientID from NATS and adds queueDir with MQTT.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/endpoint.go

@@ -29,7 +29,9 @@ import (
 
 	"github.com/minio/minio-go/pkg/set"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mem"
 	"github.com/minio/minio/pkg/mountinfo"
 )
 
@@ -198,9 +200,57 @@ func (endpoints EndpointList) GetString(i int) string {
 	return endpoints[i].String()
 }
 
-// localEndpointsPerf - returns ServerDrivesPerfInfo for only the
+// localEndpointsMemUsage - returns ServerMemUsageInfo for only the
 // local endpoints from given list of endpoints
-func localEndpointsPerf(endpoints EndpointList) ServerDrivesPerfInfo {
+func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
+	var memUsages []mem.Usage
+	var addr string
+	scratchSpace := map[string]bool{}
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			if _, ok := scratchSpace[endpoint.Host]; ok {
+				continue
+			}
+			addr = GetLocalPeer(endpoints)
+			memUsage := mem.GetUsage()
+			memUsages = append(memUsages, memUsage)
+			scratchSpace[endpoint.Host] = true
+		}
+	}
+	return ServerMemUsageInfo{
+		Addr:  addr,
+		Usage: memUsages,
+	}
+}
+
+// localEndpointsCPULoad - returns ServerCPULoadInfo for only the
+// local endpoints from given list of endpoints
+func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
+	var cpuLoads []cpu.Load
+	var addr string
+	scratchSpace := map[string]bool{}
+	for _, endpoint := range endpoints {
+		// Only proceed for local endpoints
+		if endpoint.IsLocal {
+			if _, ok := scratchSpace[endpoint.Host]; ok {
+				continue
+			}
+			addr = GetLocalPeer(endpoints)
+			cpuLoad := cpu.GetLoad()
+			cpuLoads = append(cpuLoads, cpuLoad)
+			scratchSpace[endpoint.Host] = true
+		}
+	}
+	return ServerCPULoadInfo{
+		Addr: addr,
+		Load: cpuLoads,
+	}
+}
+
+// localEndpointsDrivePerf - returns ServerDrivesPerfInfo for only the
+// local endpoints from given list of endpoints
+func localEndpointsDrivePerf(endpoints EndpointList) ServerDrivesPerfInfo {
 	var dps []disk.Performance
 	var addr string
 	for _, endpoint := range endpoints {

cmd/gateway-main.go

@@ -176,15 +176,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Add API router.
 	registerAPIRouter(router, encryptionEnabled)
 
-	// Dummy endpoint representing gateway instance.
-	globalEndpoints = []Endpoint{{
-		URL:     &url.URL{Path: "/minio/gateway"},
-		IsLocal: true,
-	}}
-
-	// Initialize Admin Peers.
-	initGlobalAdminPeers(globalEndpoints)
-
 	var getCert certs.GetCertificateFunc
 	if globalTLSCerts != nil {
 		getCert = globalTLSCerts.GetCertificate

cmd/globals.go

@@ -20,7 +20,6 @@ import (
 	"crypto/x509"
 	"fmt"
 	"os"
-	"runtime"
 	"time"
 
 	isatty "github.com/mattn/go-isatty"
@@ -162,12 +161,6 @@ var (
 	// File to log HTTP request/response headers and body.
 	globalHTTPTraceFile *os.File
 
-	// List of admin peers.
-	globalAdminPeers = adminPeers{}
-
-	// Minio server user agent string.
-	globalServerUserAgent = "Minio/" + ReleaseTag + " (" + runtime.GOOS + "; " + runtime.GOARCH + ")"
-
 	globalEndpoints EndpointList
 
 	// Global server's network statistics
@@ -361,7 +354,6 @@ func getGlobalInfo() (globalInfo map[string]interface{}) {
 		"isEnvRegion":      globalIsEnvRegion,
 		"isSSL":            globalIsSSL,
 		"serverRegion":     globalServerRegion,
-		"serverUserAgent":  globalServerUserAgent,
 		// Add more relevant global settings here.
 	}
 

cmd/local-admin-client.go

@@ -1,112 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"context"
-	"errors"
-	"os"
-
-	"io/ioutil"
-)
-
-// localAdminClient - represents admin operation to be executed locally.
-type localAdminClient struct{}
-
-// SignalService - sends a restart or stop signal to the local server
-func (lc localAdminClient) SignalService(s serviceSignal) error {
-	switch s {
-	case serviceRestart, serviceStop:
-		globalServiceSignalCh <- s
-	default:
-		return errUnsupportedSignal
-	}
-	return nil
-}
-
-// ReInitFormat - re-initialize disk format.
-func (lc localAdminClient) ReInitFormat(dryRun bool) error {
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-	return objectAPI.ReloadFormat(context.Background(), dryRun)
-}
-
-// ServerInfo - Returns the server info of this server.
-func (lc localAdminClient) ServerInfo() (sid ServerInfoData, e error) {
-	if globalBootTime.IsZero() {
-		return sid, errServerNotInitialized
-	}
-
-	// Build storage info
-	objLayer := newObjectLayerFn()
-	if objLayer == nil {
-		return sid, errServerNotInitialized
-	}
-	storage := objLayer.StorageInfo(context.Background())
-
-	return ServerInfoData{
-		StorageInfo: storage,
-		ConnStats:   globalConnStats.toServerConnStats(),
-		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
-		Properties: ServerProperties{
-			Uptime:   UTCNow().Sub(globalBootTime),
-			Version:  Version,
-			CommitID: CommitID,
-			SQSARN:   globalNotificationSys.GetARNList(),
-			Region:   globalServerConfig.GetRegion(),
-		},
-	}, nil
-}
-
-// StartProfiling - starts profiling on the local server.
-func (lc localAdminClient) StartProfiling(profiler string) error {
-	if globalProfiler != nil {
-		globalProfiler.Stop()
-	}
-	prof, err := startProfiler(profiler, "")
-	if err != nil {
-		return err
-	}
-	globalProfiler = prof
-	return nil
-}
-
-// DownloadProfilingData - stops and returns profiling data of the local server.
-func (lc localAdminClient) DownloadProfilingData() ([]byte, error) {
-	if globalProfiler == nil {
-		return nil, errors.New("profiler not enabled")
-	}
-
-	profilerPath := globalProfiler.Path()
-
-	// Stop the profiler
-	globalProfiler.Stop()
-
-	profilerFile, err := os.Open(profilerPath)
-	if err != nil {
-		return nil, err
-	}
-
-	data, err := ioutil.ReadAll(profilerFile)
-	if err != nil {
-		return nil, err
-	}
-
-	return data, nil
-}

cmd/notification.go

@@ -17,11 +17,13 @@
 package cmd
 
 import (
+	"archive/zip"
 	"bytes"
 	"context"
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"net"
 	"net/url"
 	"path"
@@ -90,85 +92,252 @@ func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string)
 	}()
 }
 
-// ReloadFormat - calls ReloadFormat RPC call on all peers.
-func (sys *NotificationSys) ReloadFormat(dryRun bool) map[xnet.Host]error {
-	errors := make(map[xnet.Host]error)
-	var wg sync.WaitGroup
-	for addr, client := range sys.peerRPCClientMap {
-		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient) {
-			defer wg.Done()
-			// Try to load format in three attempts, before giving up.
-			for i := 0; i < 3; i++ {
-				err := client.ReloadFormat(dryRun)
-				if err == nil {
-					break
+// A NotificationGroup is a collection of goroutines working on subtasks that are part of
+// the same overall task.
+//
+// A zero NotificationGroup is valid and does not cancel on error.
+type NotificationGroup struct {
+	wg   sync.WaitGroup
+	errs []NotificationPeerErr
+}
+
+// WithNPeers returns a new NotificationGroup with length of errs slice upto nerrs,
+// upon Wait() errors are returned collected from all tasks.
+func WithNPeers(nerrs int) *NotificationGroup {
+	return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs)}
+}
+
+// Wait blocks until all function calls from the Go method have returned, then
+// returns the slice of errors from all function calls.
+func (g *NotificationGroup) Wait() []NotificationPeerErr {
+	g.wg.Wait()
+	return g.errs
+}
+
+// Go calls the given function in a new goroutine.
+//
+// The first call to return a non-nil error will be
+// collected in errs slice and returned by Wait().
+func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) {
+	g.wg.Add(1)
+
+	go func() {
+		defer g.wg.Done()
+		g.errs[index] = NotificationPeerErr{
+			Host: addr,
+		}
+		for i := 0; i < 3; i++ {
+			if err := f(); err != nil {
+				g.errs[index].Err = err
+				// Last iteration log the error.
+				if i == 2 {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+					ctx := logger.SetReqInfo(ctx, reqInfo)
+					logger.LogIf(ctx, err)
 				}
-				errors[addr] = err
 				// Wait for one second and no need wait after last attempt.
 				if i < 2 {
 					time.Sleep(1 * time.Second)
 				}
+				continue
 			}
-		}(addr, client)
-	}
-	wg.Wait()
+			break
+		}
+	}()
+}
 
-	return errors
+// ReloadFormat - calls ReloadFormat RPC call on all peers.
+func (sys *NotificationSys) ReloadFormat(dryRun bool) []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.ReloadFormat(dryRun)
+		}, idx, addr)
+		idx++
+	}
+	return ng.Wait()
 }
 
 // LoadUsers - calls LoadUsers RPC call on all peers.
-func (sys *NotificationSys) LoadUsers() map[xnet.Host]error {
-	errors := make(map[xnet.Host]error)
-	var wg sync.WaitGroup
+func (sys *NotificationSys) LoadUsers() []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
 	for addr, client := range sys.peerRPCClientMap {
-		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient) {
-			defer wg.Done()
-			// Try to load users in three attempts.
-			for i := 0; i < 3; i++ {
-				err := client.LoadUsers()
-				if err == nil {
-					break
-				}
-				errors[addr] = err
-				// Wait for one second and no need wait after last attempt.
-				if i < 2 {
-					time.Sleep(1 * time.Second)
-				}
-			}
-		}(addr, client)
+		ng.Go(context.Background(), client.LoadUsers, idx, addr)
+		idx++
 	}
-	wg.Wait()
-
-	return errors
+	return ng.Wait()
 }
 
 // LoadCredentials - calls LoadCredentials RPC call on all peers.
-func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
-	errors := make(map[xnet.Host]error)
+func (sys *NotificationSys) LoadCredentials() []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		ng.Go(context.Background(), client.LoadCredentials, idx, addr)
+		idx++
+	}
+	return ng.Wait()
+}
+
+// StartProfiling - start profiling on remote peers, by initiating a remote RPC.
+func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.StartProfiling(profiler)
+		}, idx, addr)
+		idx++
+	}
+	return ng.Wait()
+}
+
+// DownloadProfilingData - download profiling data from all remote peers.
+func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) bool {
+	profilingDataFound := false
+
+	// Initialize a zip writer which will provide a zipped content
+	// of profiling data of all nodes
+	zipWriter := zip.NewWriter(writer)
+	defer zipWriter.Close()
+
+	for addr, client := range sys.peerRPCClientMap {
+		data, err := client.DownloadProfilingData()
+		if err != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, err)
+			continue
+		}
+
+		profilingDataFound = true
+
+		// Send profiling data to zip as file
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    fmt.Sprintf("profiling-%s.pprof", addr),
+			size:    int64(len(data)),
+			mode:    0600,
+			modTime: UTCNow(),
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, zerr)
+			continue
+		}
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, zerr)
+			continue
+		}
+		if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
+			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+			ctx := logger.SetReqInfo(ctx, reqInfo)
+			logger.LogIf(ctx, err)
+			continue
+		}
+	}
+
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return profilingDataFound
+	}
+
+	data, err := getProfileData()
+	if err != nil {
+		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", thisAddr.String())
+		ctx := logger.SetReqInfo(ctx, reqInfo)
+		logger.LogIf(ctx, err)
+		return profilingDataFound
+	}
+
+	profilingDataFound = true
+
+	// Send profiling data to zip as file
+	header, zerr := zip.FileInfoHeader(dummyFileInfo{
+		name:    fmt.Sprintf("profiling-%s.pprof", thisAddr),
+		size:    int64(len(data)),
+		mode:    0600,
+		modTime: UTCNow(),
+		isDir:   false,
+		sys:     nil,
+	})
+
+	zwriter, zerr := zipWriter.CreateHeader(header)
+	if zerr != nil {
+		return profilingDataFound
+	}
+
+	if _, err = io.Copy(zwriter, bytes.NewBuffer(data)); err != nil {
+		return profilingDataFound
+	}
+
+	return profilingDataFound
+}
+
+// SignalService - calls signal service RPC call on all peers.
+func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerErr {
+	var idx = 0
+	ng := WithNPeers(len(sys.peerRPCClientMap))
+	for addr, client := range sys.peerRPCClientMap {
+		client := client
+		ng.Go(context.Background(), func() error {
+			return client.SignalService(sig)
+		}, idx, addr)
+		idx++
+	}
+	return ng.Wait()
+}
+
+// ServerInfo - calls ServerInfo RPC call on all peers.
+func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
+	var idx = 0
+	serverInfo := make([]ServerInfo, len(sys.peerRPCClientMap))
 	var wg sync.WaitGroup
 	for addr, client := range sys.peerRPCClientMap {
 		wg.Add(1)
-		go func(addr xnet.Host, client *PeerRPCClient) {
+		go func(idx int, addr xnet.Host, client *PeerRPCClient) {
 			defer wg.Done()
-			// Try to load credentials in three attempts.
+			// Try to fetch serverInfo remotely in three attempts.
 			for i := 0; i < 3; i++ {
-				err := client.LoadCredentials()
+				info, err := client.ServerInfo()
 				if err == nil {
-					break
+					serverInfo[idx] = ServerInfo{
+						Addr: addr.String(),
+						Data: &info,
+					}
+					return
+				}
+				serverInfo[idx] = ServerInfo{
+					Addr:  addr.String(),
+					Data:  &info,
+					Error: err.Error(),
+				}
+				// Last iteration log the error.
+				if i == 2 {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+					ctx := logger.SetReqInfo(ctx, reqInfo)
+					logger.LogIf(ctx, err)
 				}
-				errors[addr] = err
 				// Wait for one second and no need wait after last attempt.
 				if i < 2 {
 					time.Sleep(1 * time.Second)
 				}
 			}
-		}(addr, client)
+		}(idx, addr, client)
+		idx++
 	}
 	wg.Wait()
-
-	return errors
+	return serverInfo
 }
 
 // SetBucketPolicy - calls SetBucketPolicy RPC call on all peers.
@@ -537,6 +706,56 @@ func (sys *NotificationSys) DrivePerfInfo() []ServerDrivesPerfInfo {
 	return reply
 }
 
+// MemUsageInfo - Mem utilization information
+func (sys *NotificationSys) MemUsageInfo() []ServerMemUsageInfo {
+	reply := make([]ServerMemUsageInfo, len(sys.peerRPCClientMap))
+	var wg sync.WaitGroup
+	var i int
+	for addr, client := range sys.peerRPCClientMap {
+		wg.Add(1)
+		go func(addr xnet.Host, client *PeerRPCClient, idx int) {
+			defer wg.Done()
+			memi, err := client.MemUsageInfo()
+			if err != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String())
+				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				logger.LogIf(ctx, err)
+				memi.Addr = addr.String()
+				memi.Error = err.Error()
+			}
+			reply[idx] = memi
+		}(addr, client, i)
+		i++
+	}
+	wg.Wait()
+	return reply
+}
+
+// CPULoadInfo - CPU utilization information
+func (sys *NotificationSys) CPULoadInfo() []ServerCPULoadInfo {
+	reply := make([]ServerCPULoadInfo, len(sys.peerRPCClientMap))
+	var wg sync.WaitGroup
+	var i int
+	for addr, client := range sys.peerRPCClientMap {
+		wg.Add(1)
+		go func(addr xnet.Host, client *PeerRPCClient, idx int) {
+			defer wg.Done()
+			cpui, err := client.CPULoadInfo()
+			if err != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr.String())
+				ctx := logger.SetReqInfo(context.Background(), reqInfo)
+				logger.LogIf(ctx, err)
+				cpui.Addr = addr.String()
+				cpui.Error = err.Error()
+			}
+			reply[idx] = cpui
+		}(addr, client, i)
+		i++
+	}
+	wg.Wait()
+	return reply
+}
+
 // NewNotificationSys - creates new notification system object.
 func NewNotificationSys(config *serverConfig, endpoints EndpointList) *NotificationSys {
 	targetList := getNotificationTargets(config)

cmd/object-handlers.go

@@ -179,7 +179,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		rs := &HTTPRangeSpec{
 			IsSuffixLength: isSuffixLength,
 			Start:          offset,
-			End:            length,
+			End:            offset + length,
 		}
 
 		return getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, ObjectOptions{})
@@ -653,12 +653,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// TODO: Reject requests where body/payload is present, for now we don't even read it.
 
-	// Copy source path.
-	cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
-	if err != nil {
-		// Save unescaped string as is.
-		cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
-	}
+	// Read escaped copy source path to check for parameters.
+	cpSrcPath := r.Header.Get("X-Amz-Copy-Source")
 
 	// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
 	// Regardless of whether you have enabled versioning, each object in your bucket
@@ -673,8 +669,18 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			writeErrorResponse(w, ErrNoSuchVersion, r.URL, guessIsBrowserReq(r))
 			return
 		}
+		// Note that url.Parse does the unescaping
 		cpSrcPath = u.Path
 	}
+	if vid := r.Header.Get("X-Amz-Copy-Source-Version-Id"); vid != "" {
+		// Check if versionId header was added, if yes then check if
+		// its non "null" value, we should error out since we do not support
+		// any versions other than "null".
+		if vid != "null" {
+			writeErrorResponse(w, ErrNoSuchVersion, r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
 
 	srcBucket, srcObject := path2BucketAndObject(cpSrcPath)
 	// If source object is empty or bucket is empty, reply back invalid copy source.
@@ -699,7 +705,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	}
 
 	var srcOpts, dstOpts ObjectOptions
-	srcOpts, err = copySrcEncryptionOpts(ctx, r, srcBucket, srcObject)
+	srcOpts, err := copySrcEncryptionOpts(ctx, r, srcBucket, srcObject)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -1349,6 +1355,11 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 	bucket := vars["bucket"]
 	object := vars["object"]
 
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone {
+		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	// This request header needs to be set prior to setting ObjectOptions
 	if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
 		r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
@@ -1364,11 +1375,6 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone {
-		writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
-		return
-	}
-
 	// Deny if WORM is enabled
 	if globalWORMEnabled {
 		if _, err = objectAPI.GetObjectInfo(ctx, bucket, object, opts); err == nil {
@@ -1466,12 +1472,8 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Copy source path.
-	cpSrcPath, err := url.QueryUnescape(r.Header.Get("X-Amz-Copy-Source"))
-	if err != nil {
-		// Save unescaped string as is.
-		cpSrcPath = r.Header.Get("X-Amz-Copy-Source")
-	}
+	// Read escaped copy source path to check for parameters.
+	cpSrcPath := r.Header.Get("X-Amz-Copy-Source")
 
 	// Check https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectVersioning.html
 	// Regardless of whether you have enabled versioning, each object in your bucket
@@ -1486,8 +1488,18 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			writeErrorResponse(w, ErrNoSuchVersion, r.URL, guessIsBrowserReq(r))
 			return
 		}
+		// Note that url.Parse does the unescaping
 		cpSrcPath = u.Path
 	}
+	if vid := r.Header.Get("X-Amz-Copy-Source-Version-Id"); vid != "" {
+		// Check if X-Amz-Copy-Source-Version-Id header was added, if yes then check if
+		// its non "null" value, we should error out since we do not support
+		// any versions other than "null".
+		if vid != "null" {
+			writeErrorResponse(w, ErrNoSuchVersion, r.URL, guessIsBrowserReq(r))
+			return
+		}
+	}
 
 	srcBucket, srcObject := path2BucketAndObject(cpSrcPath)
 	// If source object is empty or bucket is empty, reply back invalid copy source.

cmd/object-handlers_test.go

@@ -25,6 +25,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"runtime"
 	"strings"
 
 	"io/ioutil"
@@ -1523,14 +1524,15 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	// test cases with inputs and expected result for Copy Object.
 	testCases := []struct {
-		bucketName        string
-		copySourceHeader  string // data for "X-Amz-Copy-Source" header. Contains the object to be copied in the URL.
-		copySourceRange   string // data for "X-Amz-Copy-Source-Range" header, contains the byte range offsets of data to be copied.
-		uploadID          string // uploadID of the transaction.
-		invalidPartNumber bool   // Sets an invalid multipart.
-		maximumPartNumber bool   // Sets a maximum parts.
-		accessKey         string
-		secretKey         string
+		bucketName          string
+		copySourceHeader    string // data for "X-Amz-Copy-Source" header. Contains the object to be copied in the URL.
+		copySourceVersionID string // data for "X-Amz-Copy-Source-Version-Id" header.
+		copySourceRange     string // data for "X-Amz-Copy-Source-Range" header, contains the byte range offsets of data to be copied.
+		uploadID            string // uploadID of the transaction.
+		invalidPartNumber   bool   // Sets an invalid multipart.
+		maximumPartNumber   bool   // Sets a maximum parts.
+		accessKey           string
+		secretKey           string
 		// expected output.
 		expectedRespStatus int
 	}{
@@ -1694,6 +1696,44 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 			secretKey:          credentials.SecretKey,
 			expectedRespStatus: http.StatusOK,
 		},
+		// Test case - 14, copy part 1 from from newObject1 with null versionId
+		{
+			bucketName:         bucketName,
+			uploadID:           uploadID,
+			copySourceHeader:   url.QueryEscape("/"+bucketName+"/"+objectName) + "?versionId=null",
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 15, copy part 1 from from newObject1 with non null versionId
+		{
+			bucketName:         bucketName,
+			uploadID:           uploadID,
+			copySourceHeader:   url.QueryEscape("/"+bucketName+"/"+objectName) + "?versionId=17",
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 16, copy part 1 from from newObject1 with null X-Amz-Copy-Source-Version-Id
+		{
+			bucketName:          bucketName,
+			uploadID:            uploadID,
+			copySourceHeader:    url.QueryEscape("/" + bucketName + "/" + objectName),
+			copySourceVersionID: "null",
+			accessKey:           credentials.AccessKey,
+			secretKey:           credentials.SecretKey,
+			expectedRespStatus:  http.StatusOK,
+		},
+		// Test case - 16, copy part 1 from from newObject1 with non null X-Amz-Copy-Source-Version-Id
+		{
+			bucketName:          bucketName,
+			uploadID:            uploadID,
+			copySourceHeader:    url.QueryEscape("/" + bucketName + "/" + objectName),
+			copySourceVersionID: "17",
+			accessKey:           credentials.AccessKey,
+			secretKey:           credentials.SecretKey,
+			expectedRespStatus:  http.StatusNotFound,
+		},
 	}
 
 	for i, testCase := range testCases {
@@ -1717,6 +1757,9 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 		if testCase.copySourceHeader != "" {
 			req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
 		}
+		if testCase.copySourceVersionID != "" {
+			req.Header.Set("X-Amz-Copy-Source-Version-Id", testCase.copySourceVersionID)
+		}
 		if testCase.copySourceRange != "" {
 			req.Header.Set("X-Amz-Copy-Source-Range", testCase.copySourceRange)
 		}
@@ -1752,6 +1795,9 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 		if testCase.copySourceHeader != "" {
 			reqV2.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
 		}
+		if testCase.copySourceVersionID != "" {
+			reqV2.Header.Set("X-Amz-Copy-Source-Version-Id", testCase.copySourceVersionID)
+		}
 		if testCase.copySourceRange != "" {
 			reqV2.Header.Set("X-Amz-Copy-Source-Range", testCase.copySourceRange)
 		}
@@ -1802,7 +1848,10 @@ func TestAPICopyObjectHandler(t *testing.T) {
 func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
 	credentials auth.Credentials, t *testing.T) {
 
-	objectName := "test-object"
+	objectName := "test?object" // use file with ? to test URL parsing...
+	if runtime.GOOS == "windows" {
+		objectName = "test-object" // ...except on Windows
+	}
 	// object used for anonymous HTTP request test.
 	anonObject := "anon-object"
 	var err error
@@ -1861,6 +1910,7 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		bucketName           string
 		newObjectName        string // name of the newly copied object.
 		copySourceHeader     string // data for "X-Amz-Copy-Source" header. Contains the object to be copied in the URL.
+		copySourceVersionID  string // data for "X-Amz-Copy-Source-Version-Id" header.
 		copyModifiedHeader   string // data for "X-Amz-Copy-Source-If-Modified-Since" header
 		copyUnmodifiedHeader string // data for "X-Amz-Copy-Source-If-Unmodified-Since" header
 		metadataGarbage      bool
@@ -2071,6 +2121,44 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 			secretKey:            credentials.SecretKey,
 			expectedRespStatus:   http.StatusOK,
 		},
+		// Test case - 17, copy metadata from newObject1 with null versionId
+		{
+			bucketName:         bucketName,
+			newObjectName:      "newObject1",
+			copySourceHeader:   url.QueryEscape("/"+bucketName+"/"+objectName) + "?versionId=null",
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 18, copy metadata from newObject1 with non null versionId
+		{
+			bucketName:         bucketName,
+			newObjectName:      "newObject1",
+			copySourceHeader:   url.QueryEscape("/"+bucketName+"/"+objectName) + "?versionId=17",
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
+		// Test case - 19, copy metadata from newObject1 with null X-Amz-Copy-Source-Version-Id
+		{
+			bucketName:          bucketName,
+			newObjectName:       "newObject1",
+			copySourceHeader:    url.QueryEscape("/" + bucketName + "/" + objectName),
+			copySourceVersionID: "null",
+			accessKey:           credentials.AccessKey,
+			secretKey:           credentials.SecretKey,
+			expectedRespStatus:  http.StatusOK,
+		},
+		// Test case - 20, copy metadata from newObject1 with non null X-Amz-Copy-Source-Version-Id
+		{
+			bucketName:          bucketName,
+			newObjectName:       "newObject1",
+			copySourceHeader:    url.QueryEscape("/" + bucketName + "/" + objectName),
+			copySourceVersionID: "17",
+			accessKey:           credentials.AccessKey,
+			secretKey:           credentials.SecretKey,
+			expectedRespStatus:  http.StatusNotFound,
+		},
 	}
 
 	for i, testCase := range testCases {
@@ -2089,6 +2177,9 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		if testCase.copySourceHeader != "" {
 			req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
 		}
+		if testCase.copySourceVersionID != "" {
+			req.Header.Set("X-Amz-Copy-Source-Version-Id", testCase.copySourceVersionID)
+		}
 		if testCase.copyModifiedHeader != "" {
 			req.Header.Set("X-Amz-Copy-Source-If-Modified-Since", testCase.copyModifiedHeader)
 		}
@@ -2150,6 +2241,9 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
 		if testCase.copySourceHeader != "" {
 			reqV2.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
 		}
+		if testCase.copySourceVersionID != "" {
+			reqV2.Header.Set("X-Amz-Copy-Source-Version-Id", testCase.copySourceVersionID)
+		}
 		if testCase.copyModifiedHeader != "" {
 			reqV2.Header.Set("X-Amz-Copy-Source-If-Modified-Since", testCase.copyModifiedHeader)
 		}

cmd/peer-rpc-client.go

@@ -150,6 +150,56 @@ func (rpcClient *PeerRPCClient) DrivePerfInfo() (ServerDrivesPerfInfo, error) {
 	return reply, err
 }
 
+// MemUsageInfo - returns mem utilization info for remote server
+func (rpcClient *PeerRPCClient) MemUsageInfo() (ServerMemUsageInfo, error) {
+	args := AuthArgs{}
+	var reply ServerMemUsageInfo
+
+	err := rpcClient.Call(peerServiceName+".MemUsageInfo", &args, &reply)
+	return reply, err
+}
+
+// CPULoadInfo - returns cpu performance info for remote server
+func (rpcClient *PeerRPCClient) CPULoadInfo() (ServerCPULoadInfo, error) {
+	args := AuthArgs{}
+	var reply ServerCPULoadInfo
+
+	err := rpcClient.Call(peerServiceName+".CPULoadInfo", &args, &reply)
+	return reply, err
+}
+
+// StartProfiling - starts profiling on the remote server.
+func (rpcClient *PeerRPCClient) StartProfiling(profiler string) error {
+	args := StartProfilingArgs{Profiler: profiler}
+	reply := VoidReply{}
+	return rpcClient.Call(peerServiceName+".StartProfiling", &args, &reply)
+}
+
+// DownloadProfilingData - download already running profiling on the remote server.
+func (rpcClient *PeerRPCClient) DownloadProfilingData() ([]byte, error) {
+	args := AuthArgs{}
+	var reply []byte
+	err := rpcClient.Call(peerServiceName+".DownloadProfilingData", &args, &reply)
+	return reply, err
+}
+
+// SignalService - calls load server info RPC.
+func (rpcClient *PeerRPCClient) SignalService(sig serviceSignal) error {
+	args := SignalServiceArgs{Sig: sig}
+	reply := VoidReply{}
+
+	return rpcClient.Call(peerServiceName+".SignalService", &args, &reply)
+}
+
+// ServerInfo - calls load server info RPC.
+func (rpcClient *PeerRPCClient) ServerInfo() (ServerInfoData, error) {
+	args := AuthArgs{}
+	reply := ServerInfoData{}
+
+	err := rpcClient.Call(peerServiceName+".ServerInfo", &args, &reply)
+	return reply, err
+}
+
 // NewPeerRPCClient - returns new peer RPC client.
 func NewPeerRPCClient(host *xnet.Host) (*PeerRPCClient, error) {
 	scheme := "http"

cmd/peer-rpc-server.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"sort"
+	"time"
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/logger"
@@ -30,7 +32,7 @@ import (
 )
 
 const peerServiceName = "Peer"
-const peerServiceSubPath = "/s3/remote"
+const peerServiceSubPath = "/peer/remote"
 
 var peerServicePath = path.Join(minioReservedBucketPath, peerServiceSubPath)
 
@@ -118,7 +120,8 @@ type ListenBucketNotificationArgs struct {
 	Addr       xnet.Host      `json:"addr"`
 }
 
-// ListenBucketNotification - handles listen bucket notification RPC call. It creates PeerRPCClient target which pushes requested events to target in remote peer.
+// ListenBucketNotification - handles listen bucket notification RPC call.
+// It creates PeerRPCClient target which pushes requested events to target in remote peer.
 func (receiver *peerRPCReceiver) ListenBucketNotification(args *ListenBucketNotificationArgs, reply *VoidReply) error {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
@@ -245,7 +248,138 @@ func (receiver *peerRPCReceiver) DrivePerfInfo(args *AuthArgs, reply *ServerDriv
 		return errServerNotInitialized
 	}
 
-	*reply = localEndpointsPerf(globalEndpoints)
+	*reply = localEndpointsDrivePerf(globalEndpoints)
+	return nil
+}
+
+// CPULoadInfo - handles cpu performance RPC call
+func (receiver *peerRPCReceiver) CPULoadInfo(args *AuthArgs, reply *ServerCPULoadInfo) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+	*reply = localEndpointsCPULoad(globalEndpoints)
+	return nil
+}
+
+// MemUsageInfo - handles mem utilization RPC call
+func (receiver *peerRPCReceiver) MemUsageInfo(args *AuthArgs, reply *ServerMemUsageInfo) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+	*reply = localEndpointsMemUsage(globalEndpoints)
+	return nil
+}
+
+// uptimes - used to sort uptimes in chronological order.
+type uptimes []time.Duration
+
+func (ts uptimes) Len() int {
+	return len(ts)
+}
+
+func (ts uptimes) Less(i, j int) bool {
+	return ts[i] < ts[j]
+}
+
+func (ts uptimes) Swap(i, j int) {
+	ts[i], ts[j] = ts[j], ts[i]
+}
+
+// getPeerUptimes - returns the uptime.
+func getPeerUptimes(serverInfo []ServerInfo) time.Duration {
+	// In a single node Erasure or FS backend setup the uptime of
+	// the setup is the uptime of the single minio server
+	// instance.
+	if !globalIsDistXL {
+		return UTCNow().Sub(globalBootTime)
+	}
+
+	var times []time.Duration
+
+	for _, info := range serverInfo {
+		if info.Error != "" {
+			continue
+		}
+		times = append(times, info.Data.Properties.Uptime)
+	}
+
+	// Sort uptimes in chronological order.
+	sort.Sort(uptimes(times))
+
+	// Return the latest time as the uptime.
+	return times[0]
+}
+
+// StartProfilingArgs - holds the RPC argument for StartingProfiling RPC call
+type StartProfilingArgs struct {
+	AuthArgs
+	Profiler string
+}
+
+// StartProfiling - profiling server receiver.
+func (receiver *peerRPCReceiver) StartProfiling(args *StartProfilingArgs, reply *VoidReply) error {
+	if globalProfiler != nil {
+		globalProfiler.Stop()
+	}
+	var err error
+	globalProfiler, err = startProfiler(args.Profiler, "")
+	return err
+}
+
+// DownloadProfilingData - download profiling data.
+func (receiver *peerRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[]byte) error {
+	var err error
+	*reply, err = getProfileData()
+	return err
+}
+
+var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
+
+// SignalServiceArgs - send event RPC arguments.
+type SignalServiceArgs struct {
+	AuthArgs
+	Sig serviceSignal
+}
+
+// SignalService - signal service receiver.
+func (receiver *peerRPCReceiver) SignalService(args *SignalServiceArgs, reply *VoidReply) error {
+	switch args.Sig {
+	case serviceRestart, serviceStop:
+		globalServiceSignalCh <- args.Sig
+	default:
+		return errUnsupportedSignal
+	}
+	return nil
+}
+
+// ServerInfo - server info receiver.
+func (receiver *peerRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoData) error {
+	if globalBootTime.IsZero() {
+		return errServerNotInitialized
+	}
+
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return errServerNotInitialized
+	}
+
+	// Server info data.
+	*reply = ServerInfoData{
+		StorageInfo: objLayer.StorageInfo(context.Background()),
+		ConnStats:   globalConnStats.toServerConnStats(),
+		HTTPStats:   globalHTTPStats.toServerHTTPStats(),
+		Properties: ServerProperties{
+			Uptime:   UTCNow().Sub(globalBootTime),
+			Version:  Version,
+			CommitID: CommitID,
+			SQSARN:   globalNotificationSys.GetARNList(),
+			Region:   globalServerConfig.GetRegion(),
+		},
+	}
+
 	return nil
 }
 

cmd/routers.go

@@ -41,7 +41,7 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
 	// Register distributed namespace lock.
 	registerDistNSLockRouter(router)
 
-	// Register S3 peer communication router.
+	// Register peer communication router.
 	registerPeerRPCRouter(router)
 }
 
@@ -104,9 +104,6 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
 	// Add STS router always.
 	registerSTSRouter(router)
 
-	// Add Admin RPC router
-	registerAdminRPCRouter(router)
-
 	// Add Admin router, all APIs are enabled in server mode.
 	registerAdminRouter(router, true)
 

cmd/server-main.go

@@ -293,9 +293,6 @@ func serverMain(ctx *cli.Context) {
 		logger.Fatal(uiErrUnexpectedError(err), "Unable to configure one of server's RPC services")
 	}
 
-	// Initialize Admin Peers inter-node communication only in distributed setup.
-	initGlobalAdminPeers(globalEndpoints)
-
 	var getCert certs.GetCertificateFunc
 	if globalTLSCerts != nil {
 		getCert = globalTLSCerts.GetCertificate

cmd/sts-handlers.go

@@ -123,10 +123,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *h
 	}
 
 	// Notify all other Minio peers to reload temp users
-	for host, err := range globalNotificationSys.LoadUsers() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 
@@ -207,10 +207,10 @@ func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *
 	}
 
 	// Notify all other Minio peers to reload temp users
-	for host, err := range globalNotificationSys.LoadUsers() {
-		if err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
-			logger.LogIf(ctx, err)
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
 		}
 	}
 

cmd/utils.go

@@ -208,6 +208,26 @@ func (p profilerWrapper) Path() string {
 	return p.pathFn()
 }
 
+// Returns current profile data, returns error if there is no active
+// profiling in progress. Stops an active profile.
+func getProfileData() ([]byte, error) {
+	if globalProfiler == nil {
+		return nil, errors.New("profiler not enabled")
+	}
+
+	profilerPath := globalProfiler.Path()
+
+	// Stop the profiler
+	globalProfiler.Stop()
+
+	profilerFile, err := os.Open(profilerPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return ioutil.ReadAll(profilerFile)
+}
+
 // Starts a profiler returns nil if profiler is not enabled, caller needs to handle this.
 func startProfiler(profilerType, dirPath string) (interface {
 	Stop()

cmd/web-handlers.go

@@ -752,10 +752,10 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
 
 	if errs := globalNotificationSys.LoadCredentials(); len(errs) != 0 {
 		reply.PeerErrMsgs = make(map[string]string)
-		for host, err := range errs {
-			err = fmt.Errorf("Unable to update credentials on server %v: %v", host, err)
+		for _, nerr := range errs {
+			err = fmt.Errorf("Unable to update credentials on server %v: %v", nerr.Host, nerr.Err)
 			logger.LogIf(context.Background(), err)
-			reply.PeerErrMsgs[host.String()] = err.Error()
+			reply.PeerErrMsgs[nerr.Host.String()] = err.Error()
 		}
 	} else {
 		reply.Token, err = authenticateWeb(creds.AccessKey, creds.SecretKey)

docs/bucket/notifications/README.md

@@ -163,6 +163,7 @@ The Minio server configuration file is stored on the backend in json format. The
 | `clientId` | _string_ | Unique ID for the MQTT broker to identify Minio |
 | `username` | _string_ | Username to connect to the MQTT server (if required) |
 | `password` | _string_ | Password to connect to the MQTT server (if required) |
+| `queueDir` | _string_ | Persistent store for events when MQTT broker is offline |
 
 An example configuration for MQTT is shown below:
 
@@ -175,10 +176,14 @@ An example configuration for MQTT is shown below:
         "qos": 1,
         "clientId": "minio",
         "username": "",
-        "password": ""
+        "password": "",
+        "queueDir": ""
     }
 }
 ```
+MQTT supports persistent event store in the client-side. The persistent store will backup events when the MQTT broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queueDir` field in the mqtt config. For eg, the `queueDir` can be `/home/events`.
+
+
 To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment in json format, and save it locally.
 ```sh
 $ mc admin config get myminio/ > /tmp/myconfig
@@ -1031,7 +1036,7 @@ An example configuration for NSQ is shown below:
         "nsqdAddress": "127.0.0.1:4150",
         "topic": "minio",
         "tls": {
-            "enable": false,       
+            "enable": false,
             "skipVerify": true
         }
     }

docs/config/config.sample.json

@@ -86,7 +86,8 @@
 				"username": "",
 				"password": "",
 				"reconnectInterval": 0,
-				"keepAliveInterval": 0
+				"keepAliveInterval": 0,
+				"queueDir": ""
 			}
 		},
 		"mysql": {

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+  image: minio/minio:RELEASE.2019-01-10T00-21-20Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+  image: minio/minio:RELEASE.2019-01-10T00-21-20Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+  image: minio/minio:RELEASE.2019-01-10T00-21-20Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+  image: minio/minio:RELEASE.2019-01-10T00-21-20Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+    image: minio/minio:RELEASE.2019-01-10T00-21-20Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+        image: minio/minio:RELEASE.2019-01-10T00-21-20Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+        image: minio/minio:RELEASE.2019-01-10T00-21-20Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+        image: minio/minio:RELEASE.2019-01-10T00-21-20Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2018-12-27T18-33-08Z
+        image: minio/minio:RELEASE.2019-01-10T00-21-20Z
         args:
         - server
         - /data

docs/security/README.md

@@ -0,0 +1,248 @@
+# Minio Security Overview [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
+
+## Server-Side Encryption
+
+Minio supports two different types of server-side encryption ([SSE](#sse)):
+ - **SSE-C**: The Minio server en/decrypts an object with a secret key provided by the S3 client
+              as part of the HTTP request headers. Therefore, [SSE-C](#ssec) requires TLS/HTTPS.
+ - **SSE-S3**: The Minio server en/decrypts an object with a secret key managed by a KMS.
+               Therefore, Minio requires a valid KMS configuration for [SSE-S3](#sses3).
+
+### Server-Side Encryption - Preliminaries
+
+#### Secret Keys
+
+The Minio server uses an unique, randomly generated secret key per object also known as,
+Object Encryption Key ([OEK](#oek)). Neither the client-provided SSE-C key nor the KMS-managed
+key is directly used to en/decrypt an object. Instead, the OEK is stored as part of the object
+metadata next to the object in an encrypted form. To en/decrypt the OEK another secret key is
+needed also known as, Key Encryption Key ([KEK](#kek)).  
+
+The Minio server runs a key-derivation algorithm to generate the KEK using a pseudo-random 
+function ([PRF](#prf)):  
+`KEK := PRF(EK, IV, context_values)` where
+ - [EK](#ek): is the external key. In case of SSE-C this is the client-provided key. In case of SSE-S3
+              this is secret key generated by the KMS. For further details see 
+              [SSE-C](#Server-Side-Encryption-with-client-provided-Keys) or 
+              [SSE-S3](#Server-Side-Encryption-with-a-KMS).
+ - [IV](#iv): is a randomly generated initialization vector. It is public and part of the 
+              object metadata. 
+ - `context_values`: are values like the bucket and object name and other information which should be
+                     cryptographically bound to the KEK.
+
+To summarize for any encrypted object there exists (at least) three different keys:
+ 1. [OEK](#oek): A secret and unique key used to encrypted the object, stored in an encrypted form as 
+     part of the object metadata and only loaded to RAM in plaintext during en/decrypting the object. 
+ 2. [KEK](#kek): A secret and unique key used to en/decrypt the OEK and never stored anywhere. 
+    It is(re-)generated whenever en/decrypting an object using an external secret key and public 
+    parameters.
+ 3. [EK](#ek): An external secret key - either the SSE-C client-provided key or a secret key 
+    generated by the KMS.
+
+#### Content Encryption
+
+The Minio server uses an authenticated encryption scheme ([AEAD](#aead)) to en/decrypt and 
+authenticate the object content. The AEAD is combined with some state to build a 
+*Secure Channel*. A *Secure Channel* is a cryptographic construction that ensures confidentiality 
+and integrity of the processed data. In particular the *Secure Channel* splits the plaintext content 
+into fixed size chunks and en/decrypts each chunk separately using an unique key-nonce combination.
+
+```
+plaintext   := chunk_0          ||       chunk_1          ||       chunk_2          ||       ...
+                 |                         |                         |
+                 |                         |                         |
+               AEAD <- key, nonce + 0    AEAD <- key, nonce + 1    AEAD <- key, nonce + 2    ...
+                 |                         |                         |
+                 |                         |                         |
+ciphertext  := sealed_chunk_0   ||       sealed_chunk_1   ||       sealed_chunk_2   ||       ...
+```
+<center>Figure 1 - Secure Channel construction</center>
+
+In case of a S3 multi-part operation each part is en/decrypted with the scheme shown in 
+Figure 1. However, for each part an unique secret key is derived from the OEK and the part
+number using a PRF. So in case of multi-part not the OEK but the output of `PRF(OEK, part_id)` 
+is used as secret key.
+
+#### Cryptographic Primitives
+
+The SSE schemes described in [Secret Keys](#Secret-Keys) and [Content Encryption](#Content-Encryption)
+are generic over the cryptographic primitives. However, the Minio server uses the following
+cryptographic primitive implementations:
+ - [PRF](#prf): HMAC-SHA-256
+ - [AEAD](#aead): AES-256-GCM if the CPU supports AES-NI, ChaCha20-Poly1305 otherwise.
+           More specifically AES-256-GCM is only selected for X86-64 CPUs with AES-NI extension.
+
+Further any secret key (apart from the KMS-generated ones) is 256 bits long. The KMS-generated keys
+may be 256 bits but this depends on the KMS capabilities and configuration.
+
+The *Secure Channel* splits the object content into chunks of a fixed size of `65536` bytes. The last
+chunk may be smaller to avoid adding additional overhead and is treated specially to prevent truncation 
+attacks. The nonce value is 96 bits long and generated randomly per object / multi-part part. The 
+*Secure Channel* supports plaintexts up to `65536 * 2^32 = 256 TiB`. 
+
+#### Randomness
+
+The Minio server generates unique keys and other cryptographic values using a cryptographically
+secure pseudo-random number generator ([CSPRNG](#csprng)). However, in the context of SSE,
+the Minio server does not require that the CSPRNG generates values that are indistinguishable
+from truly random bit strings. Instead, it is sufficient if the generated values are unique - which
+is a weaker requirement. Nevertheless other parts - for example the TLS-stack - may require that
+CSPRNG-generated values are indistinguishable from truly random bit strings.
+
+### Server-Side Encryption with client-provided Keys
+
+SSE-C allows an S3 client to en/decrypt an object at the Minio server. Therefore the S3 client
+sends a secret key as part of the HTTP request. This secret key is **never** stored by the
+Minio server and only resides in RAM during the en/decryption process. 
+
+Minio does not assume or require that the client-provided key is unique. It may be used for
+multiple objects or buckets. Especially a single client-provided key may be used for all 
+objects - even though all objects must be treated as compromised if that key is ever compromised.
+
+#### Key rotation
+
+S3 clients can change the client-provided key of an existing object. Therefore an S3 client
+must perform a S3 COPY operation where the copy source and destination are equal. Further the
+COPY request headers must contain the current and the new client key:
+ - `X-Amz-Server-Side-Encryption-Customer-Key`: Base64 encoded new key.
+ - `X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key`: Base64 encoded current key.
+
+Such a special COPY request is also known as S3 SSE-C key rotation.
+
+### Server-Side Encryption with a KMS
+
+SSE-S3 allows an S3 client to en/decrypt an object at the Minio server using a KMS. The Minio 
+server only assumes that the KMS provides two services:
+ 1. `GenerateKey`: Takes a key ID and generates a new data key from a master key referenced by
+                   the key ID. It returns the new data key in two different forms: The plain data key
+                   and the data key encrypted using the master key. 
+ 2. `DecryptKey`: Takes a key ID and an encrypted data key and returns the plain data key - the 
+                  decryption of the encrypted data key using the master key referenced by the key ID -
+                  on success or an error otherwise.
+
+More details about supported KMS implementations and configuration can be found at the [KMS guide](https://github.com/minio/minio/blob/master/docs/kms/README.md).
+
+The Minio server requests a new data key from the KMS for each uploaded object and uses that data key
+as EK. Additionally it stores the encrypted form of the data key and the master key ID as part
+of the object metadata. The plain data only resides in RAM during the en/decryption process.
+The Minio server does not store any SSE-related key at the KMS. Instead the KMS is treated as trusted
+component that performs key sealing/unsealing operations to build a key hierarchy:
+
+```
+                                                          CMK (master key)
+                                                           |
+                       +-----------------------------------+-----------------------------------+
+                       |                                   |                                   |
+               +-------+----------------+          +-------+----------------+                 ... 
+               |  EK_1 | EK_1_encrypted |          |  EK_2 | EK_2_encrypted |    
+               +---+----------+---------+          +---+----------+---------+   
+                   |          |                        |          |
+                   |          |                        |          |
+               +---+---+      |                    +---+---+      |
+               | KEK_1 |      |                    | KEK_2 |      |
+               +---+---+      |                    +---+---+      |
+                   |          |                        |          |
+                   |          |                        |          |
+               +---+---+      |                    +---+---+      |
+               | OEK_1 |      |                    | OEK_2 |      |
+               +---+---+      |                    +---+---+      |
+                              |                                   |
+                              |                                   |
+                              |                                   |
+                    +---------+---------+               +---------+---------+
+                    | object_metadata_1 |               | object_metadata_2 |
+                    +-------------------+               +-------------------+
+```
+<center>Figure 2 - KMS key hierarchy</center>
+
+
+#### Key rotation - Basic Operation
+
+The Minio server supports key rotation for SSE-S3 encrypted objects. Therefore, an S3 client 
+must perform a S3 COPY operation where the copy source and destination are equal and the SSE-S3 HTTP 
+header is set. The minio server decrypts the OEK using the current encrypted data key and the
+master key ID of the object metadata. If this succeeds, the server requests a new data key
+from the KMS using the master key ID of the **current Minio KMS configuration** and re-wraps the
+*OEK* with a new *KEK* derived from the new data key / EK:
+
+```
+              object metadata                                         KMS
+                    |                                                  |
+                    |                     +----------------+        1a |  +-------+
+                    |-------------------->| EK_1_encrypted |-----------|->| CMK_1 |
+                    |                     +----------------+           |  +---+---+
+                    |                                                  |      |
+                    |              +---------------+       +------+ 1b |      |
+                    |------------->| OEK_encrypted |       | EK_1 |<---|------+
+                    |              +-------+-------+       +------+    |
+                    |                       \             /            |
+                    |                        \___  2  ___/             |
+                    |                            \___/                 |
+                    |                              |                   |
+                    |                           +--+--+                |
+                    |                           | OEK |                |  +-------+
+                    |                           +--+--+                |  | CMK_2 |
+                    |                              |                   |  +---+---+
+                    |                              |                   |      |
+                    | 5     +----------------+     |4      +------+ 3a |      |
+                    |<------| OEK_encrypted' |<----+-------| EK_2 |<---|------+
+                    |       +----------------+             +------+    |      |
+                    |                    +----------------+         3b |      |
+                    |<-------------------| EK_2_encrypted |<-----------|------+
+                    |                    +----------------+            |
+                    |                                                  |
+
+
+1a) Send encrypted data key and master key ID to KMS.
+1b) Receive decrypted data key.
+2)  Decrypt encrypted object key with the KEK derived from the data key.
+3a) Receive new plain data key from the KMS using the master key ID of the server config.
+3b) Receive encrypted form of the data key from the KMS.
+4)  Derive a new KEK from the new data key and re-encrypt the OEK with it.
+5)  Store the encrypted OEK encrypted data key and master key ID in object metadata.
+ ```
+<center>Figure 3 - KMS data key rotation</center>
+
+#### Key rotation - Extensions
+
+The basic SSE-S3 key rotation operation can be used to build more powerful key management 
+operations. The following options are possible to perform manually but do not have fully 
+functional API's at this time.
+
+1. **Master key migration**: The [SSE-S3 key rotation](#Key-rotation---Basic-Operation) can be performed
+                             on multiple/all objects to move them from one to another master key.
+2. **Secure object erasure**: The [SSE-S3 key rotation](#Key-rotation---Basic-Operation) can be applied
+                              to one/multiple objects with a randomly generated master key which is
+                              not stored at the KMS. That leads to an encrypted data key which can
+                              never be decrypted anymore.
+3. **Periodical key migration**: The [SSE-S3 key rotation](#Key-rotation---Basic-Operation) can be 
+                                 invoked after a certain time period to migrate one or more objects
+                                 from one master key to another.
+
+#### Secure Erasure and Locking
+
+The Minio server requires an available KMS to en/decrypt SSE-S3 encrypted objects. Therefore it
+is possible to erase or lock some or all encrypted objects. For example in case of a detected attack
+or other emergency situations the following actions can be taken:
+ - Seal the KMS such that it cannot be accessed by Minio server anymore. That will lock **all**
+   SSE-S3 encrypted objects protected by master keys stored on the KMS. All these objects
+   can not be decrypted as long as the KMS is sealed.
+ - Seal/Unmount one/some master keys. That will lock all SSE-S3 encrypted objects protected by
+   these master keys. All these objects can not be decrypted as long as the key(s) are sealed.
+ - Delete one/some master keys. From a security standpoint, this is equal to erasing all SSE-S3
+   encrypted objects protected by these master keys. All these objects are lost forever as they cannot 
+   be decrypted. Especially deleting all master keys at the KMS is equivalent to secure erasing all 
+   SSE-S3 encrypted objects.
+
+## Acronyms
+
+ - <a name="aead"></a>**AEAD**: Authenticated Encryption with Associated Data
+ - <a name="csprng"></a>**CSPRNG**: Cryptographically Secure Pseudo Random Number Generator 
+ - <a name="ek"></a>**EK**: External Key
+ - <a name="iv"></a>**IV**: Initialization Vector
+ - <a name="kek"></a>**KEK**: Key Encryption Key
+ - <a name="oek"></a>**OEK**: Object Encryption Key
+ - <a name="prf"></a>**PRF**: Pseudo Random Function
+ - <a name="sse"></a>**SSE**: Server-Side Encryption
+ - <a name="ssec"></a>**SSE-C**: Server-Side Encryption with client-provided Keys
+ - <a name="sses3"></a>**SSE-S3**: Server-Side Encryption with a KMS

docs/sts/client-grants.md

@@ -24,6 +24,15 @@ The OAuth 2.0 access token that is provided by the identity provider. Applicatio
 | *Length Constraints* | *Minimum length of 4. Maximum length of 2048.* |
 | *Required* | *Yes* |
 
+#### Version
+Indicates STS API version information, the only supported value is '2011-06-15'.  This value is borrowed from AWS STS API documentation for compatibility reasons.
+
+
+| Params | Value |
+| :-- | :-- |
+| *Type* | *String* |
+| *Required* | *Yes* |
+
 #### Response Elements
 XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_ResponseElements)
 
@@ -32,7 +41,7 @@ XML error response for this API is similar to [AWS STS AssumeRoleWithWebIdentity
 
 #### Sample Request
 ```
-http://minio.cluster:9000?Action=AssumeRoleWithClientGrants&DurationSeconds=3600&Token=eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTU0MTgwOTU4MiwiaWF0IjoxNTQxODA1OTgyLCJqdGkiOiI2Y2YyMGIwZS1lNGZmLTQzZmQtYTdiYS1kYTc3YTE3YzM2MzYifQ.Jm29jPliRvrK6Os34nSK3rhzIYLFjE__zdVGNng3uGKXGKzP3We_i6NPnhA0szJXMOKglXzUF1UgSz8MctbaxFS8XDusQPVe4LkB_45hwBm6TmBxzui911nt-1RbBLN_jZIlvl2lPrbTUH5hSn9kEkph6seWanTNQpz9tNEoVa6R_OX3kpJqxe8tLQUWw453A1JTwFNhdHa6-f1K8_Q_eEZ_4gOYINQ9t_fhTibdbkXZkJQFLop-Jwoybi9s4nwQU_dATocgcufq5eCeNItQeleT-23lGxIz0X7CiJrJynYLdd-ER0F77SumqEb5iCxhxuf4H7dovwd1kAmyKzLxpw
+http://minio.cluster:9000?Action=AssumeRoleWithClientGrants&DurationSeconds=3600&Token=eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTU0MTgwOTU4MiwiaWF0IjoxNTQxODA1OTgyLCJqdGkiOiI2Y2YyMGIwZS1lNGZmLTQzZmQtYTdiYS1kYTc3YTE3YzM2MzYifQ.Jm29jPliRvrK6Os34nSK3rhzIYLFjE__zdVGNng3uGKXGKzP3We_i6NPnhA0szJXMOKglXzUF1UgSz8MctbaxFS8XDusQPVe4LkB_45hwBm6TmBxzui911nt-1RbBLN_jZIlvl2lPrbTUH5hSn9kEkph6seWanTNQpz9tNEoVa6R_OX3kpJqxe8tLQUWw453A1JTwFNhdHa6-f1K8_Q_eEZ_4gOYINQ9t_fhTibdbkXZkJQFLop-Jwoybi9s4nwQU_dATocgcufq5eCeNItQeleT-23lGxIz0X7CiJrJynYLdd-ER0F77SumqEb5iCxhxuf4H7dovwd1kAmyKzLxpw&Version=2011-06-15
 ```
 
 #### Sample Response

docs/sts/web-identity.md

@@ -22,6 +22,14 @@ The OAuth 2.0 access token that is provided by the web identity provider. Applic
 | *Length Constraints* | *Minimum length of 4. Maximum length of 2048.* |
 | *Required* | *Yes* |
 
+#### Version
+Indicates STS API version information, the only supported value is '2011-06-15'. This value is borrowed from AWS STS API documentation for compatibility reasons.
+
+| Params | Value |
+| :-- | :-- |
+| *Type* | *String* |
+| *Required* | *Yes* |
+
 #### Response Elements
 XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_ResponseElements)
 
@@ -30,7 +38,7 @@ XML error response for this API is similar to [AWS STS AssumeRoleWithWebIdentity
 
 #### Sample Request
 ```
-http://minio.cluster:9000?Action=AssumeRoleWithWebIdentity&DurationSeconds=3600&WebIdentityToken=eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTU0MTgwOTU4MiwiaWF0IjoxNTQxODA1OTgyLCJqdGkiOiI2Y2YyMGIwZS1lNGZmLTQzZmQtYTdiYS1kYTc3YTE3YzM2MzYifQ.Jm29jPliRvrK6Os34nSK3rhzIYLFjE__zdVGNng3uGKXGKzP3We_i6NPnhA0szJXMOKglXzUF1UgSz8MctbaxFS8XDusQPVe4LkB_45hwBm6TmBxzui911nt-1RbBLN_jZIlvl2lPrbTUH5hSn9kEkph6seWanTNQpz9tNEoVa6R_OX3kpJqxe8tLQUWw453A1JTwFNhdHa6-f1K8_Q_eEZ_4gOYINQ9t_fhTibdbkXZkJQFLop-Jwoybi9s4nwQU_dATocgcufq5eCeNItQeleT-23lGxIz0X7CiJrJynYLdd-ER0F77SumqEb5iCxhxuf4H7dovwd1kAmyKzLxpw
+http://minio.cluster:9000?Action=AssumeRoleWithWebIdentity&DurationSeconds=3600&WebIdentityToken=eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTU0MTgwOTU4MiwiaWF0IjoxNTQxODA1OTgyLCJqdGkiOiI2Y2YyMGIwZS1lNGZmLTQzZmQtYTdiYS1kYTc3YTE3YzM2MzYifQ.Jm29jPliRvrK6Os34nSK3rhzIYLFjE__zdVGNng3uGKXGKzP3We_i6NPnhA0szJXMOKglXzUF1UgSz8MctbaxFS8XDusQPVe4LkB_45hwBm6TmBxzui911nt-1RbBLN_jZIlvl2lPrbTUH5hSn9kEkph6seWanTNQpz9tNEoVa6R_OX3kpJqxe8tLQUWw453A1JTwFNhdHa6-f1K8_Q_eEZ_4gOYINQ9t_fhTibdbkXZkJQFLop-Jwoybi9s4nwQU_dATocgcufq5eCeNItQeleT-23lGxIz0X7CiJrJynYLdd-ER0F77SumqEb5iCxhxuf4H7dovwd1kAmyKzLxpw&Version=2011-06-15
 ```
 
 #### Sample Response

pkg/cpu/counter_darwin.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,16 +14,17 @@
  * limitations under the License.
  */
 
-package cmd
+package cpu
 
 import (
-	"testing"
+	"errors"
+	"time"
 )
 
-func TestLocalAdminClientSignalService(t *testing.T) {
-	testAdminCmdRunnerSignalService(t, &localAdminClient{})
+func newCounter() (counter, error) {
+	return counter{}, errors.New("cpu metrics not implemented for darwin platform")
 }
 
-func TestLocalAdminClientServerInfo(t *testing.T) {
-	testAdminCmdRunnerServerInfo(t, &localAdminClient{})
+func (c counter) now() time.Time {
+	return time.Time{}
 }

pkg/cpu/counter_linux.go

@@ -0,0 +1,40 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"syscall"
+	"time"
+	"unsafe"
+)
+
+const (
+	// ProcessClock corresponds to the High-resolution per-process
+	// timer from the CPU represented in stdlib as CLOCK_PROCESS_CPUTIME_ID
+	processClock = 2
+)
+
+func newCounter() (counter, error) {
+	return counter{}, nil
+}
+
+func (c counter) now() time.Time {
+	var ts syscall.Timespec
+	syscall.Syscall(syscall.SYS_CLOCK_GETTIME, processClock, uintptr(unsafe.Pointer(&ts)), 0)
+	sec, nsec := ts.Unix()
+	return time.Unix(sec, nsec)
+}

pkg/cpu/counter_windows.go

@@ -0,0 +1,30 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"errors"
+	"time"
+)
+
+func newCounter() (counter, error) {
+	return counter{}, errors.New("cpu metrics not implemented for windows platform")
+}
+
+func (c counter) now() time.Time {
+	return time.Time{}
+}

pkg/cpu/cpu.go

@@ -0,0 +1,91 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"fmt"
+	"sync"
+	"time"
+)
+
+const (
+	// cpuLoadWindow is the interval of time for which the
+	// cpu utilization is measured
+	cpuLoadWindow = 200 * time.Millisecond
+
+	// cpuLoadSampleSize is the number of samples measured
+	// for calculating cpu utilization
+	cpuLoadSampleSize = 3
+
+	// endOfTime represents the end of time
+	endOfTime = time.Duration(1<<63 - 1)
+)
+
+// Load holds CPU utilization % measured in three intervals of 200ms each
+type Load struct {
+	Avg   string `json:"avg"`
+	Max   string `json:"max"`
+	Min   string `json:"min"`
+	Error string `json:"error,omitempty"`
+}
+
+type counter struct{}
+
+// GetLoad returns the CPU utilization % of the current process
+func GetLoad() Load {
+	vals := make(chan time.Duration, 3)
+	wg := sync.WaitGroup{}
+	for i := 0; i < cpuLoadSampleSize; i++ {
+		cpuCounter, err := newCounter()
+		if err != nil {
+			return Load{
+				Error: err.Error(),
+			}
+		}
+		wg.Add(1)
+		go func() {
+			start := cpuCounter.now()
+			time.Sleep(cpuLoadWindow)
+			end := cpuCounter.now()
+			vals <- end.Sub(start)
+			wg.Done()
+		}()
+	}
+	wg.Wait()
+
+	sum := time.Duration(0)
+	max := time.Duration(0)
+	min := (endOfTime)
+	for i := 0; i < cpuLoadSampleSize; i++ {
+		val := <-vals
+		sum = sum + val
+		if val > max {
+			max = val
+		}
+		if val < min {
+			min = val
+		}
+	}
+	close(vals)
+	avg := sum / 3
+	return Load{
+		Avg:   fmt.Sprintf("%.2f%%", toFixed4(float64(avg)/float64(200*time.Millisecond))*100),
+		Max:   fmt.Sprintf("%.2f%%", toFixed4(float64(max)/float64(200*time.Millisecond))*100),
+		Min:   fmt.Sprintf("%.2f%%", toFixed4(float64(min)/float64(200*time.Millisecond))*100),
+		Error: "",
+	}
+}

pkg/cpu/helpers.go

@@ -0,0 +1,30 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cpu
+
+import (
+	"math"
+)
+
+func round(num float64) int {
+	return int(num + math.Copysign(0.5, num))
+}
+
+func toFixed4(num float64) float64 {
+	output := math.Pow(10, float64(4))
+	return float64(round(num*output)) / output
+}

pkg/event/target/mqtt.go

@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"errors"
 	"net/url"
+	"path/filepath"
 	"time"
 
 	"github.com/eclipse/paho.mqtt.golang"
@@ -41,6 +42,7 @@ type MQTTArgs struct {
 	MaxReconnectInterval time.Duration  `json:"reconnectInterval"`
 	KeepAlive            time.Duration  `json:"keepAliveInterval"`
 	RootCAs              *x509.CertPool `json:"-"`
+	QueueDir             string         `json:"queueDir"`
 }
 
 // Validate MQTTArgs fields
@@ -57,6 +59,9 @@ func (m MQTTArgs) Validate() error {
 	default:
 		return errors.New("unknown protocol in broker address")
 	}
+	if m.QueueDir != "" && !filepath.IsAbs(m.QueueDir) {
+		return errors.New("queueDir path should be absolute")
+	}
 	return nil
 }
 
@@ -96,10 +101,14 @@ func (target *MQTTTarget) Send(eventData event.Event) error {
 
 	token := target.client.Publish(target.args.Topic, target.args.QoS, false, string(data))
 
-	if token.Wait() {
+	// In-case if the queueDir is configured OR mqtt broker is offline, the token.Wait() waits indefinitely
+	if target.args.QueueDir == "" {
+		token.Wait()
 		return token.Error()
 	}
 
+	// Need a fix from the paho.mqtt.golang library - https://github.com/eclipse/paho.mqtt.golang/issues/274
+	// Right now the server panics on IO File store errors, returning nil for now.
 	return nil
 }
 
@@ -120,6 +129,10 @@ func NewMQTTTarget(id string, args MQTTArgs) (*MQTTTarget, error) {
 		SetTLSConfig(&tls.Config{RootCAs: args.RootCAs}).
 		AddBroker(args.Broker.String())
 
+	if args.QueueDir != "" {
+		options = options.SetStore(mqtt.NewFileStore(args.QueueDir))
+	}
+
 	client := mqtt.NewClient(options)
 	token := client.Connect()
 	if token.Wait() && token.Error() != nil {

pkg/event/target/webhook.go

@@ -82,7 +82,6 @@ func (target *WebhookTarget) Send(eventData event.Event) error {
 		return err
 	}
 
-	// req.Header.Set("User-Agent", globalServerUserAgent)
 	req.Header.Set("Content-Type", "application/json")
 
 	resp, err := target.httpClient.Do(req)

pkg/madmin/API.md

@@ -39,7 +39,7 @@ func main() {
 | Service operations         | Info operations  | Healing operations                    | Config operations        | IAM operations | Misc                                |
 |:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------|:------------------------------------|
 | [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) |
-| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
+| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`ServerCPULoadInfo`](#ServerCPULoadInfo) | [`ServerMemUsageInfo`](#ServerMemUsageInfo)  | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
 | | |            | [`GetConfigKeys`](#GetConfigKeys) | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) |
 | | |            | [`SetConfigKeys`](#SetConfigKeys) | [`AddCannedPolicy`](#AddCannedPolicy) | |
 
@@ -222,6 +222,40 @@ Fetches drive performance information for all cluster nodes. Returned value is i
 |`disk.Performance.WriteSpeed` | _float64_ | Write speed on above path in Bytes/s. |
 |`disk.Performance.ReadSpeed` | _float64_ | Read speed on above path in Bytes/s. |
 
+<a name="ServerCPULoadInfo"></a>
+### ServerCPULoadInfo() ([]ServerCPULoadInfo, error)
+
+Fetches CPU utilization for all cluster nodes. Returned value is in Bytes.
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`cpui.Addr` | _string_ | Address of the server the following information  is retrieved from. |
+|`cpui.Error` | _string_ | Errors (if any) encountered while reaching this node |
+|`cpui.CPULoad` | _cpu.Load_ | The load on the CPU. |
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`cpu.Load.Avg` | _string_ | The average utilization % of the CPU measured in a 200ms interval |
+|`cpu.Load.Min` | _string_ | The minimum utilization % of the CPU measured in a 200ms interval |
+|`cpu.Load.Max` | _string_ | The maximum utilization % of the CPU measured in a 200ms interval |
+|`cpu.Load.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
+
+<a name="ServerMemUsageInfo"></a>
+### ServerMemUsageInfo() ([]ServerMemUsageInfo, error)
+
+Fetches Mem utilization for all cluster nodes. Returned value is in Bytes.
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`memi.Addr` | _string_ | Address of the server the following information  is retrieved from. |
+|`memi.Error` | _string_ | Errors (if any) encountered while reaching this node |
+|`memi.MemUsage` | _mem.Usage_ | The utilitzation of Memory |
+
+| Param | Type | Description |
+|-------|------|-------------|
+|`mem.Usage.Mem` | _string_ | The total number of bytes obtained from the OS |
+|`mem.Usage.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
+
 ## 6. Heal operations
 
 <a name="Heal"></a>

pkg/madmin/examples/cpu-load-info.go

@@ -0,0 +1,44 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	st, err := madmClnt.ServerCPULoadInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Println(st)
+}

pkg/madmin/examples/mem-usage-info.go

@@ -0,0 +1,44 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY and my-bucketname are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	st, err := madmClnt.ServerMemUsageInfo()
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Println(st)
+}

pkg/madmin/info-commands.go

@@ -24,7 +24,9 @@ import (
 	"net/url"
 	"time"
 
+	"github.com/minio/minio/pkg/cpu"
 	"github.com/minio/minio/pkg/disk"
+	"github.com/minio/minio/pkg/mem"
 )
 
 // BackendType - represents different backend types.
@@ -193,3 +195,89 @@ func (adm *AdminClient) ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) {
 
 	return info, nil
 }
+
+// ServerCPULoadInfo holds information about address and cpu load of
+// a single server node
+type ServerCPULoadInfo struct {
+	Addr  string     `json:"addr"`
+	Error string     `json:"error,omitempty"`
+	Load  []cpu.Load `json:"load"`
+}
+
+// ServerCPULoadInfo - Returns cpu utilization information
+func (adm *AdminClient) ServerCPULoadInfo() ([]ServerCPULoadInfo, error) {
+	v := url.Values{}
+	v.Set("perfType", string("cpu"))
+	resp, err := adm.executeMethod("GET", requestData{
+		relPath:     "/v1/performance",
+		queryValues: v,
+	})
+
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check response http status code
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	// Unmarshal the server's json response
+	var info []ServerCPULoadInfo
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(respBytes, &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}
+
+// ServerMemUsageInfo holds information about address and memory utilization of
+// a single server node
+type ServerMemUsageInfo struct {
+	Addr  string      `json:"addr"`
+	Error string      `json:"error,omitempty"`
+	Usage []mem.Usage `json:"usage"`
+}
+
+// ServerMemUsageInfo - Returns mem utilization information
+func (adm *AdminClient) ServerMemUsageInfo() ([]ServerMemUsageInfo, error) {
+	v := url.Values{}
+	v.Set("perfType", string("mem"))
+	resp, err := adm.executeMethod("GET", requestData{
+		relPath:     "/v1/performance",
+		queryValues: v,
+	})
+
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check response http status code
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	// Unmarshal the server's json response
+	var info []ServerMemUsageInfo
+
+	respBytes, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = json.Unmarshal(respBytes, &info)
+	if err != nil {
+		return nil, err
+	}
+
+	return info, nil
+}

pkg/mem/mem.go

@@ -0,0 +1,39 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package mem
+
+import (
+	"runtime"
+
+	humanize "github.com/dustin/go-humanize"
+)
+
+// Usage holds memory utilization information in human readable format
+type Usage struct {
+	Mem   string `json:"mem"`
+	Error string `json:"error,omitempty"`
+}
+
+// GetUsage measures the total memory provisioned for the current process
+// from the OS
+func GetUsage() Usage {
+	memStats := new(runtime.MemStats)
+	runtime.ReadMemStats(memStats)
+	return Usage{
+		Mem: humanize.IBytes(memStats.Sys),
+	}
+}

pkg/s3select/json/record.go

@@ -36,6 +36,8 @@ type Record struct {
 func (r *Record) Get(name string) (*sql.Value, error) {
 	result := gjson.GetBytes(r.data, name)
 	switch result.Type {
+	case gjson.Null:
+		return sql.NewNull(), nil
 	case gjson.False:
 		return sql.NewBool(false), nil
 	case gjson.Number:
@@ -53,6 +55,8 @@ func (r *Record) Get(name string) (*sql.Value, error) {
 func (r *Record) Set(name string, value *sql.Value) (err error) {
 	var v interface{}
 	switch value.Type() {
+	case sql.Null:
+		v = value.NullValue()
 	case sql.Bool:
 		v = value.BoolValue()
 	case sql.Int:

pkg/s3select/parquet/reader.go

@@ -44,6 +44,14 @@ func (r *Reader) Read() (sql.Record, error) {
 
 	record := json.NewRecord()
 	for name, v := range parquetRecord {
+		if v.Value == nil {
+			if err = record.Set(name, sql.NewNull()); err != nil {
+				return nil, errParquetParsingError(err)
+			}
+
+			continue
+		}
+
 		var value *sql.Value
 		switch v.Type {
 		case parquetgen.Type_BOOLEAN:

pkg/s3select/select_test.go

@@ -72,12 +72,13 @@ func TestCSVINput(t *testing.T) {
 `)
 
 	var csvData = []byte(`one,two,three
-10,true,"foo"
--3,false,"bar baz"
+-1,foo,true
+,bar,false
+2.5,baz,true
 `)
 
 	var expectedResult = []byte{
-		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 49, 48, 44, 116, 114, 117, 101, 44, 102, 111, 111, 10, 225, 160, 249, 157, 0, 0, 0, 118, 0, 0, 0, 85, 8, 177, 111, 125, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 51, 44, 102, 97, 108, 115, 101, 44, 98, 97, 114, 32, 98, 97, 122, 10, 120, 72, 77, 126, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 52, 55, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 52, 55, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 50, 57, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 214, 225, 163, 199, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
+		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 155, 11, 218, 231, 0, 0, 0, 112, 0, 0, 0, 85, 135, 241, 154, 221, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 146, 219, 90, 187, 0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 172, 136, 9, 26, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 53, 48, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 53, 48, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 253, 105, 8, 216, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
 	}
 
 	s3Select, err := NewS3Select(bytes.NewReader(requestXML))
@@ -122,12 +123,13 @@ func TestJSONInput(t *testing.T) {
 </SelectObjectContentRequest>
 `)
 
-	var jsonData = []byte(`{"one":10,"two":true,"three":"foo"}
-{"one":-3,"two":true,"three":"bar baz"}
+	var jsonData = []byte(`{"three":true,"two":"foo","one":-1}
+{"three":false,"two":"bar","one":null}
+{"three":true,"two":"baz","one":2.5}
 `)
 
 	var expectedResult = []byte{
-		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 49, 48, 44, 116, 114, 117, 101, 44, 102, 111, 111, 10, 225, 160, 249, 157, 0, 0, 0, 117, 0, 0, 0, 85, 79, 17, 21, 173, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 51, 44, 116, 114, 117, 101, 44, 98, 97, 114, 32, 98, 97, 122, 10, 34, 12, 125, 218, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 55, 54, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 55, 54, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 50, 56, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 124, 107, 174, 242, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
+		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 155, 11, 218, 231, 0, 0, 0, 112, 0, 0, 0, 85, 135, 241, 154, 221, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 146, 219, 90, 187, 0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 172, 136, 9, 26, 0, 0, 0, 237, 0, 0, 0, 67, 90, 179, 204, 45, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 49, 49, 50, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 49, 49, 50, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 181, 40, 50, 250, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
 	}
 
 	s3Select, err := NewS3Select(bytes.NewReader(requestXML))
@@ -195,7 +197,7 @@ func TestParquetInput(t *testing.T) {
 	}
 
 	var expectedResult = []byte{
-		0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 50, 46, 53, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 209, 8, 249, 77, 0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 45, 143, 126, 67, 0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 230, 139, 42, 176, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 56, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 199, 176, 2, 83, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
+		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 155, 11, 218, 231, 0, 0, 0, 112, 0, 0, 0, 85, 135, 241, 154, 221, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 146, 219, 90, 187, 0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 172, 136, 9, 26, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 128, 96, 253, 66, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
 	}
 
 	s3Select, err := NewS3Select(bytes.NewReader(requestXML))

pkg/s3select/sql/value.go

@@ -58,6 +58,10 @@ func (value *Value) String() string {
 
 // CSVString - encodes to CSV string.
 func (value *Value) CSVString() string {
+	if value.valueType == Null {
+		return ""
+	}
+
 	return fmt.Sprintf("%v", value.value)
 }
 
@@ -66,6 +70,15 @@ func (value *Value) MarshalJSON() ([]byte, error) {
 	return json.Marshal(value.value)
 }
 
+// NullValue - returns underlying null value. It panics if value is not null type.
+func (value *Value) NullValue() *struct{} {
+	if value.valueType == Null {
+		return nil
+	}
+
+	panic(fmt.Sprintf("requested bool value but found %T type", value.value))
+}
+
 // BoolValue - returns underlying bool value. It panics if value is not Bool type.
 func (value *Value) BoolValue() bool {
 	if value.valueType == Bool {

vendor/github.com/eclipse/paho.mqtt.golang/README.md

@@ -1,3 +1,7 @@
+
+[![GoDoc](https://godoc.org/github.com/eclipse/paho.mqtt.golang?status.svg)](https://godoc.org/github.com/eclipse/paho.mqtt.golang)
+[![Go Report Card](https://goreportcard.com/badge/github.com/eclipse/paho.mqtt.golang)](https://goreportcard.com/report/github.com/eclipse/paho.mqtt.golang)
+
 Eclipse Paho MQTT Go client
 ===========================
 
@@ -18,11 +22,12 @@ This client is designed to work with the standard Go tools, so installation is a
 go get github.com/eclipse/paho.mqtt.golang
 ```
 
-The client depends on Google's [websockets](https://godoc.org/golang.org/x/net/websocket) package, 
-also easily installed with the command:
+The client depends on Google's [websockets](https://godoc.org/golang.org/x/net/websocket) and [proxy](https://godoc.org/golang.org/x/net/proxy) package, 
+also easily installed with the commands:
 
 ```
 go get golang.org/x/net/websocket
+go get golang.org/x/net/proxy
 ```
 
 

vendor/github.com/eclipse/paho.mqtt.golang/client.go

@@ -12,6 +12,8 @@
  *    Mike Robertson
  */
 
+// Portions copyright © 2018 TIBCO Software Inc.
+
 // Package mqtt provides an MQTT v3.1.1 client library.
 package mqtt
 
@@ -19,16 +21,16 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/eclipse/paho.mqtt.golang/packets"
 )
 
-type connStatus uint
-
 const (
-	disconnected connStatus = iota
+	disconnected uint32 = iota
 	connecting
 	reconnecting
 	connected
@@ -53,18 +55,50 @@ const (
 // Numerous connection options may be specified by configuring a
 // and then supplying a ClientOptions type.
 type Client interface {
+	// IsConnected returns a bool signifying whether
+	// the client is connected or not.
 	IsConnected() bool
+	// IsConnectionOpen return a bool signifying wether the client has an active
+	// connection to mqtt broker, i.e not in disconnected or reconnect mode
+	IsConnectionOpen() bool
+	// Connect will create a connection to the message broker, by default
+	// it will attempt to connect at v3.1.1 and auto retry at v3.1 if that
+	// fails
 	Connect() Token
+	// Disconnect will end the connection with the server, but not before waiting
+	// the specified number of milliseconds to wait for existing work to be
+	// completed.
 	Disconnect(quiesce uint)
+	// Publish will publish a message with the specified QoS and content
+	// to the specified topic.
+	// Returns a token to track delivery of the message to the broker
 	Publish(topic string, qos byte, retained bool, payload interface{}) Token
+	// Subscribe starts a new subscription. Provide a MessageHandler to be executed when
+	// a message is published on the topic provided, or nil for the default handler
 	Subscribe(topic string, qos byte, callback MessageHandler) Token
+	// SubscribeMultiple starts a new subscription for multiple topics. Provide a MessageHandler to
+	// be executed when a message is published on one of the topics provided, or nil for the
+	// default handler
 	SubscribeMultiple(filters map[string]byte, callback MessageHandler) Token
+	// Unsubscribe will end the subscription from each of the topics provided.
+	// Messages published to those topics from other clients will no longer be
+	// received.
 	Unsubscribe(topics ...string) Token
+	// AddRoute allows you to add a handler for messages on a specific topic
+	// without making a subscription. For example having a different handler
+	// for parts of a wildcard subscription
 	AddRoute(topic string, callback MessageHandler)
+	// OptionsReader returns a ClientOptionsReader which is a copy of the clientoptions
+	// in use by the client.
+	OptionsReader() ClientOptionsReader
 }
 
 // client implements the Client interface
 type client struct {
+	lastSent        int64
+	lastReceived    int64
+	pingOutstanding int32
+	status          uint32
 	sync.RWMutex
 	messageIds
 	conn            net.Conn
@@ -78,10 +112,6 @@ type client struct {
 	stop            chan struct{}
 	persist         Store
 	options         ClientOptions
-	pingResp        chan struct{}
-	packetResp      chan struct{}
-	keepaliveReset  chan struct{}
-	status          connStatus
 	workers         sync.WaitGroup
 }
 
@@ -99,21 +129,26 @@ func NewClient(o *ClientOptions) Client {
 	switch c.options.ProtocolVersion {
 	case 3, 4:
 		c.options.protocolVersionExplicit = true
+	case 0x83, 0x84:
+		c.options.protocolVersionExplicit = true
 	default:
 		c.options.ProtocolVersion = 4
 		c.options.protocolVersionExplicit = false
 	}
 	c.persist = c.options.Store
 	c.status = disconnected
-	c.messageIds = messageIds{index: make(map[uint16]Token)}
+	c.messageIds = messageIds{index: make(map[uint16]tokenCompletor)}
 	c.msgRouter, c.stopRouter = newRouter()
-	c.msgRouter.setDefaultHandler(c.options.DefaultPublishHander)
+	c.msgRouter.setDefaultHandler(c.options.DefaultPublishHandler)
 	if !c.options.AutoReconnect {
 		c.options.MessageChannelDepth = 0
 	}
 	return c
 }
 
+// AddRoute allows you to add a handler for messages on a specific topic
+// without making a subscription. For example having a different handler
+// for parts of a wildcard subscription
 func (c *client) AddRoute(topic string, callback MessageHandler) {
 	if callback != nil {
 		c.msgRouter.addRoute(topic, callback)
@@ -125,54 +160,81 @@ func (c *client) AddRoute(topic string, callback MessageHandler) {
 func (c *client) IsConnected() bool {
 	c.RLock()
 	defer c.RUnlock()
+	status := atomic.LoadUint32(&c.status)
 	switch {
-	case c.status == connected:
+	case status == connected:
 		return true
-	case c.options.AutoReconnect && c.status > disconnected:
+	case c.options.AutoReconnect && status > connecting:
 		return true
 	default:
 		return false
 	}
 }
 
-func (c *client) connectionStatus() connStatus {
+// IsConnectionOpen return a bool signifying whether the client has an active
+// connection to mqtt broker, i.e not in disconnected or reconnect mode
+func (c *client) IsConnectionOpen() bool {
 	c.RLock()
 	defer c.RUnlock()
-	return c.status
+	status := atomic.LoadUint32(&c.status)
+	switch {
+	case status == connected:
+		return true
+	default:
+		return false
+	}
 }
 
-func (c *client) setConnected(status connStatus) {
+func (c *client) connectionStatus() uint32 {
+	c.RLock()
+	defer c.RUnlock()
+	status := atomic.LoadUint32(&c.status)
+	return status
+}
+
+func (c *client) setConnected(status uint32) {
 	c.Lock()
 	defer c.Unlock()
-	c.status = status
+	atomic.StoreUint32(&c.status, uint32(status))
 }
 
 //ErrNotConnected is the error returned from function calls that are
 //made when the client is not connected to a broker
 var ErrNotConnected = errors.New("Not Connected")
 
-// Connect will create a connection to the message broker
-// If clean session is false, then a slice will
-// be returned containing Receipts for all messages
-// that were in-flight at the last disconnect.
-// If clean session is true, then any existing client
-// state will be removed.
+// Connect will create a connection to the message broker, by default
+// it will attempt to connect at v3.1.1 and auto retry at v3.1 if that
+// fails
 func (c *client) Connect() Token {
 	var err error
 	t := newToken(packets.Connect).(*ConnectToken)
 	DEBUG.Println(CLI, "Connect()")
 
+	c.obound = make(chan *PacketAndToken, c.options.MessageChannelDepth)
+	c.oboundP = make(chan *PacketAndToken, c.options.MessageChannelDepth)
+	c.ibound = make(chan packets.ControlPacket)
+
 	go func() {
 		c.persist.Open()
 
 		c.setConnected(connecting)
+		c.errors = make(chan error, 1)
+		c.stop = make(chan struct{})
+
 		var rc byte
-		cm := newConnectMsgFromOptions(&c.options)
+		protocolVersion := c.options.ProtocolVersion
+
+		if len(c.options.Servers) == 0 {
+			t.setError(fmt.Errorf("No servers defined to connect to"))
+			return
+		}
 
 		for _, broker := range c.options.Servers {
+			cm := newConnectMsgFromOptions(&c.options, broker)
+			c.options.ProtocolVersion = protocolVersion
 		CONN:
 			DEBUG.Println(CLI, "about to write new connect msg")
-			c.conn, err = openConnection(broker, &c.options.TLSConfig, c.options.ConnectTimeout)
+			c.conn, err = openConnection(broker, c.options.TLSConfig, c.options.ConnectTimeout, c.options.HTTPHeaders)
 			if err == nil {
 				DEBUG.Println(CLI, "socket connected to broker")
 				switch c.options.ProtocolVersion {
@@ -180,6 +242,14 @@ func (c *client) Connect() Token {
 					DEBUG.Println(CLI, "Using MQTT 3.1 protocol")
 					cm.ProtocolName = "MQIsdp"
 					cm.ProtocolVersion = 3
+				case 0x83:
+					DEBUG.Println(CLI, "Using MQTT 3.1b protocol")
+					cm.ProtocolName = "MQIsdp"
+					cm.ProtocolVersion = 0x83
+				case 0x84:
+					DEBUG.Println(CLI, "Using MQTT 3.1.1b protocol")
+					cm.ProtocolName = "MQTT"
+					cm.ProtocolVersion = 0x84
 				default:
 					DEBUG.Println(CLI, "Using MQTT 3.1.1 protocol")
 					c.options.ProtocolVersion = 4
@@ -188,7 +258,7 @@ func (c *client) Connect() Token {
 				}
 				cm.Write(c.conn)
 
-				rc = c.connect()
+				rc, t.sessionPresent = c.connect()
 				if rc != packets.Accepted {
 					if c.conn != nil {
 						c.conn.Close()
@@ -215,32 +285,27 @@ func (c *client) Connect() Token {
 
 		if c.conn == nil {
 			ERROR.Println(CLI, "Failed to connect to a broker")
-			t.returnCode = rc
-			if rc != packets.ErrNetworkError {
-				t.err = packets.ConnErrors[rc]
-			} else {
-				t.err = fmt.Errorf("%s : %s", packets.ConnErrors[rc], err)
-			}
 			c.setConnected(disconnected)
 			c.persist.Close()
-			t.flowComplete()
+			t.returnCode = rc
+			if rc != packets.ErrNetworkError {
+				t.setError(packets.ConnErrors[rc])
+			} else {
+				t.setError(fmt.Errorf("%s : %s", packets.ConnErrors[rc], err))
+			}
 			return
 		}
 
+		c.options.protocolVersionExplicit = true
+
 		if c.options.KeepAlive != 0 {
+			atomic.StoreInt32(&c.pingOutstanding, 0)
+			atomic.StoreInt64(&c.lastReceived, time.Now().Unix())
+			atomic.StoreInt64(&c.lastSent, time.Now().Unix())
 			c.workers.Add(1)
 			go keepalive(c)
 		}
 
-		c.obound = make(chan *PacketAndToken, c.options.MessageChannelDepth)
-		c.oboundP = make(chan *PacketAndToken, c.options.MessageChannelDepth)
-		c.ibound = make(chan packets.ControlPacket)
-		c.errors = make(chan error, 1)
-		c.stop = make(chan struct{})
-		c.pingResp = make(chan struct{}, 1)
-		c.packetResp = make(chan struct{}, 1)
-		c.keepaliveReset = make(chan struct{}, 1)
-
 		c.incomingPubChan = make(chan *packets.PublishPacket, c.options.MessageChannelDepth)
 		c.msgRouter.matchAndDispatch(c.incomingPubChan, c.options.Order, c)
 
@@ -250,22 +315,19 @@ func (c *client) Connect() Token {
 			go c.options.OnConnect(c)
 		}
 
-		// Take care of any messages in the store
-		//var leftovers []Receipt
-		if c.options.CleanSession == false {
-			//leftovers = c.resume()
-		} else {
-			c.persist.Reset()
-		}
-
+		c.workers.Add(4)
 		go errorWatch(c)
-
-		// Do not start incoming until resume has completed
-		c.workers.Add(3)
 		go alllogic(c)
 		go outgoing(c)
 		go incoming(c)
 
+		// Take care of any messages in the store
+		if c.options.CleanSession == false {
+			c.resume(c.options.ResumeSubs)
+		} else {
+			c.persist.Reset()
+		}
+
 		DEBUG.Println(CLI, "exit startClient")
 		t.flowComplete()
 	}()
@@ -283,28 +345,33 @@ func (c *client) reconnect() {
 	)
 
 	for rc != 0 && c.status != disconnected {
-		cm := newConnectMsgFromOptions(&c.options)
-
 		for _, broker := range c.options.Servers {
-		CONN:
+			cm := newConnectMsgFromOptions(&c.options, broker)
 			DEBUG.Println(CLI, "about to write new connect msg")
-			c.conn, err = openConnection(broker, &c.options.TLSConfig, c.options.ConnectTimeout)
+			c.conn, err = openConnection(broker, c.options.TLSConfig, c.options.ConnectTimeout, c.options.HTTPHeaders)
 			if err == nil {
 				DEBUG.Println(CLI, "socket connected to broker")
 				switch c.options.ProtocolVersion {
+				case 0x83:
+					DEBUG.Println(CLI, "Using MQTT 3.1b protocol")
+					cm.ProtocolName = "MQIsdp"
+					cm.ProtocolVersion = 0x83
+				case 0x84:
+					DEBUG.Println(CLI, "Using MQTT 3.1.1b protocol")
+					cm.ProtocolName = "MQTT"
+					cm.ProtocolVersion = 0x84
 				case 3:
 					DEBUG.Println(CLI, "Using MQTT 3.1 protocol")
 					cm.ProtocolName = "MQIsdp"
 					cm.ProtocolVersion = 3
 				default:
 					DEBUG.Println(CLI, "Using MQTT 3.1.1 protocol")
-					c.options.ProtocolVersion = 4
 					cm.ProtocolName = "MQTT"
 					cm.ProtocolVersion = 4
 				}
 				cm.Write(c.conn)
 
-				rc = c.connect()
+				rc, _ = c.connect()
 				if rc != packets.Accepted {
 					c.conn.Close()
 					c.conn = nil
@@ -313,11 +380,6 @@ func (c *client) reconnect() {
 						ERROR.Println(CLI, "Connecting to", broker, "CONNACK was not Accepted, but rather", packets.ConnackReturnCodes[rc])
 						continue
 					}
-					if c.options.ProtocolVersion == 4 {
-						DEBUG.Println(CLI, "Trying reconnect using MQTT 3.1 protocol")
-						c.options.ProtocolVersion = 3
-						goto CONN
-					}
 				}
 				break
 			} else {
@@ -339,64 +401,69 @@ func (c *client) reconnect() {
 		}
 	}
 	// Disconnect() must have been called while we were trying to reconnect.
-	if c.status == disconnected {
+	if c.connectionStatus() == disconnected {
 		DEBUG.Println(CLI, "Client moved to disconnected state while reconnecting, abandoning reconnect")
 		return
 	}
 
+	c.stop = make(chan struct{})
+
 	if c.options.KeepAlive != 0 {
+		atomic.StoreInt32(&c.pingOutstanding, 0)
+		atomic.StoreInt64(&c.lastReceived, time.Now().Unix())
+		atomic.StoreInt64(&c.lastSent, time.Now().Unix())
 		c.workers.Add(1)
 		go keepalive(c)
 	}
 
-	c.stop = make(chan struct{})
-
 	c.setConnected(connected)
 	DEBUG.Println(CLI, "client is reconnected")
 	if c.options.OnConnect != nil {
 		go c.options.OnConnect(c)
 	}
 
+	c.workers.Add(4)
 	go errorWatch(c)
-
-	c.workers.Add(3)
 	go alllogic(c)
 	go outgoing(c)
 	go incoming(c)
+
+	c.resume(false)
 }
 
 // This function is only used for receiving a connack
 // when the connection is first started.
 // This prevents receiving incoming data while resume
 // is in progress if clean session is false.
-func (c *client) connect() byte {
+func (c *client) connect() (byte, bool) {
 	DEBUG.Println(NET, "connect started")
 
 	ca, err := packets.ReadPacket(c.conn)
 	if err != nil {
 		ERROR.Println(NET, "connect got error", err)
-		return packets.ErrNetworkError
+		return packets.ErrNetworkError, false
 	}
 	if ca == nil {
 		ERROR.Println(NET, "received nil packet")
-		return packets.ErrNetworkError
+		return packets.ErrNetworkError, false
 	}
 
 	msg, ok := ca.(*packets.ConnackPacket)
 	if !ok {
 		ERROR.Println(NET, "received msg that was not CONNACK")
-		return packets.ErrNetworkError
+		return packets.ErrNetworkError, false
 	}
 
 	DEBUG.Println(NET, "received connack")
-	return msg.ReturnCode
+	return msg.ReturnCode, msg.SessionPresent
 }
 
 // Disconnect will end the connection with the server, but not before waiting
 // the specified number of milliseconds to wait for existing work to be
 // completed.
 func (c *client) Disconnect(quiesce uint) {
-	if c.status == connected {
+	status := atomic.LoadUint32(&c.status)
+	if status == connected {
 		DEBUG.Println(CLI, "disconnecting")
 		c.setConnected(disconnected)
 
@@ -409,6 +476,7 @@ func (c *client) Disconnect(quiesce uint) {
 	} else {
 		WARN.Println(CLI, "Disconnect() called but not connected (disconnected/reconnecting)")
 		c.setConnected(disconnected)
+		return
 	}
 
 	c.disconnect()
@@ -456,7 +524,9 @@ func (c *client) closeStop() {
 	case <-c.stop:
 		DEBUG.Println("In disconnect and stop channel is already closed")
 	default:
-		close(c.stop)
+		if c.stop != nil {
+			close(c.stop)
+		}
 	}
 }
 
@@ -472,6 +542,7 @@ func (c *client) disconnect() {
 	c.closeStop()
 	c.closeConn()
 	c.workers.Wait()
+	c.messageIds.cleanUp()
 	close(c.stopRouter)
 	DEBUG.Println(CLI, "disconnected")
 	c.persist.Close()
@@ -507,13 +578,17 @@ func (c *client) Publish(topic string, qos byte, retained bool, payload interfac
 		return token
 	}
 
-	DEBUG.Println(CLI, "sending publish message, topic:", topic)
 	if pub.Qos != 0 && pub.MessageID == 0 {
 		pub.MessageID = c.getID(token)
 		token.messageID = pub.MessageID
 	}
 	persistOutbound(c.persist, pub)
-	c.obound <- &PacketAndToken{p: pub, t: token}
+	if c.connectionStatus() == reconnecting {
+		DEBUG.Println(CLI, "storing publish message (reconnecting), topic:", topic)
+	} else {
+		DEBUG.Println(CLI, "sending publish message, topic:", topic)
+		c.obound <- &PacketAndToken{p: pub, t: token}
+	}
 	return token
 }
 
@@ -536,6 +611,10 @@ func (c *client) Subscribe(topic string, qos byte, callback MessageHandler) Toke
 	sub.Qoss = append(sub.Qoss, qos)
 	DEBUG.Println(CLI, sub.String())
 
+	if strings.HasPrefix(topic, "$share") {
+		topic = strings.Join(strings.Split(topic, "/")[2:], "/")
+	}
+
 	if callback != nil {
 		c.msgRouter.addRoute(topic, callback)
 	}
@@ -575,6 +654,61 @@ func (c *client) SubscribeMultiple(filters map[string]byte, callback MessageHand
 	return token
 }
 
+// Load all stored messages and resend them
+// Call this to ensure QOS > 1,2 even after an application crash
+func (c *client) resume(subscription bool) {
+
+	storedKeys := c.persist.All()
+	for _, key := range storedKeys {
+		packet := c.persist.Get(key)
+		details := packet.Details()
+		if isKeyOutbound(key) {
+			switch packet.(type) {
+			case *packets.SubscribePacket:
+				if subscription {
+					DEBUG.Println(STR, fmt.Sprintf("loaded pending subscribe (%d)", details.MessageID))
+					token := newToken(packets.Subscribe).(*SubscribeToken)
+					c.oboundP <- &PacketAndToken{p: packet, t: token}
+				}
+			case *packets.UnsubscribePacket:
+				if subscription {
+					DEBUG.Println(STR, fmt.Sprintf("loaded pending unsubscribe (%d)", details.MessageID))
+					token := newToken(packets.Unsubscribe).(*UnsubscribeToken)
+					c.oboundP <- &PacketAndToken{p: packet, t: token}
+				}
+			case *packets.PubrelPacket:
+				DEBUG.Println(STR, fmt.Sprintf("loaded pending pubrel (%d)", details.MessageID))
+				select {
+				case c.oboundP <- &PacketAndToken{p: packet, t: nil}:
+				case <-c.stop:
+				}
+			case *packets.PublishPacket:
+				token := newToken(packets.Publish).(*PublishToken)
+				token.messageID = details.MessageID
+				c.claimID(token, details.MessageID)
+				DEBUG.Println(STR, fmt.Sprintf("loaded pending publish (%d)", details.MessageID))
+				DEBUG.Println(STR, details)
+				c.obound <- &PacketAndToken{p: packet, t: token}
+			default:
+				ERROR.Println(STR, "invalid message type in store (discarded)")
+				c.persist.Del(key)
+			}
+		} else {
+			switch packet.(type) {
+			case *packets.PubrelPacket, *packets.PublishPacket:
+				DEBUG.Println(STR, fmt.Sprintf("loaded pending incomming (%d)", details.MessageID))
+				select {
+				case c.ibound <- packet:
+				case <-c.stop:
+				}
+			default:
+				ERROR.Println(STR, "invalid message type in store (discarded)")
+				c.persist.Del(key)
+			}
+		}
+	}
+}
+
 // Unsubscribe will end the subscription from each of the topics provided.
 // Messages published to those topics from other clients will no longer be
 // received.
@@ -599,6 +733,8 @@ func (c *client) Unsubscribe(topics ...string) Token {
 	return token
 }
 
+// OptionsReader returns a ClientOptionsReader which is a copy of the clientoptions
+// in use by the client.
 func (c *client) OptionsReader() ClientOptionsReader {
 	r := ClientOptionsReader{options: &c.options}
 	return r

vendor/github.com/eclipse/paho.mqtt.golang/filestore.go

@@ -18,6 +18,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"sort"
 	"sync"
 
 	"github.com/eclipse/paho.mqtt.golang/packets"
@@ -150,13 +151,18 @@ func (store *FileStore) Reset() {
 
 // lockless
 func (store *FileStore) all() []string {
+	var err error
+	var keys []string
+	var files fileInfos
+
 	if !store.opened {
 		ERROR.Println(STR, "Trying to use file store, but not open")
 		return nil
 	}
-	keys := []string{}
-	files, rderr := ioutil.ReadDir(store.directory)
-	chkerr(rderr)
+
+	files, err = ioutil.ReadDir(store.directory)
+	chkerr(err)
+	sort.Sort(files)
 	for _, f := range files {
 		DEBUG.Println(STR, "file in All():", f.Name())
 		name := f.Name()
@@ -233,3 +239,17 @@ func exists(file string) bool {
 	}
 	return true
 }
+
+type fileInfos []os.FileInfo
+
+func (f fileInfos) Len() int {
+	return len(f)
+}
+
+func (f fileInfos) Swap(i, j int) {
+	f[i], f[j] = f[j], f[i]
+}
+
+func (f fileInfos) Less(i, j int) bool {
+	return f[i].ModTime().Before(f[j].ModTime())
+}

vendor/github.com/eclipse/paho.mqtt.golang/memstore.go

@@ -109,7 +109,7 @@ func (store *MemoryStore) Del(key string) {
 	if m == nil {
 		WARN.Println(STR, "memorystore del: message", mid, "not found")
 	} else {
-		store.messages[key] = nil
+		delete(store.messages, key)
 		DEBUG.Println(STR, "memorystore del: message", mid, "was deleted")
 	}
 }

vendor/github.com/eclipse/paho.mqtt.golang/message.go

@@ -15,7 +15,10 @@
 package mqtt
 
 import (
+	"net/url"
+
 	"github.com/eclipse/paho.mqtt.golang/packets"
+	"sync"
 )
 
 // Message defines the externals that a message implementation must support
@@ -28,6 +31,7 @@ type Message interface {
 	Topic() string
 	MessageID() uint16
 	Payload() []byte
+	Ack()
 }
 
 type message struct {
@@ -37,6 +41,8 @@ type message struct {
 	topic     string
 	messageID uint16
 	payload   []byte
+	once      sync.Once
+	ack       func()
 }
 
 func (m *message) Duplicate() bool {
@@ -63,7 +69,11 @@ func (m *message) Payload() []byte {
 	return m.payload
 }
 
-func messageFromPublish(p *packets.PublishPacket) Message {
+func (m *message) Ack() {
+	m.once.Do(m.ack)
+}
+
+func messageFromPublish(p *packets.PublishPacket, ack func()) Message {
 	return &message{
 		duplicate: p.Dup,
 		qos:       p.Qos,
@@ -71,10 +81,11 @@ func messageFromPublish(p *packets.PublishPacket) Message {
 		topic:     p.TopicName,
 		messageID: p.MessageID,
 		payload:   p.Payload,
+		ack:       ack,
 	}
 }
 
-func newConnectMsgFromOptions(options *ClientOptions) *packets.ConnectPacket {
+func newConnectMsgFromOptions(options *ClientOptions, broker *url.URL) *packets.ConnectPacket {
 	m := packets.NewControlPacket(packets.Connect).(*packets.ConnectPacket)
 
 	m.CleanSession = options.CleanSession
@@ -88,17 +99,29 @@ func newConnectMsgFromOptions(options *ClientOptions) *packets.ConnectPacket {
 		m.WillMessage = options.WillPayload
 	}
 
-	if options.Username != "" {
+	username := options.Username
+	password := options.Password
+	if broker.User != nil {
+		username = broker.User.Username()
+		if pwd, ok := broker.User.Password(); ok {
+			password = pwd
+		}
+	}
+	if options.CredentialsProvider != nil {
+		username, password = options.CredentialsProvider()
+	}
+
+	if username != "" {
 		m.UsernameFlag = true
-		m.Username = options.Username
+		m.Username = username
 		//mustn't have password without user as well
-		if options.Password != "" {
+		if password != "" {
 			m.PasswordFlag = true
-			m.Password = []byte(options.Password)
+			m.Password = []byte(password)
 		}
 	}
 
-	m.Keepalive = uint16(options.KeepAlive.Seconds())
+	m.Keepalive = uint16(options.KeepAlive)
 
 	return m
 }

vendor/github.com/eclipse/paho.mqtt.golang/messageids.go

@@ -17,6 +17,7 @@ package mqtt
 import (
 	"fmt"
 	"sync"
+	"time"
 )
 
 // MId is 16 bit message id as specified by the MQTT spec.
@@ -26,7 +27,7 @@ type MId uint16
 
 type messageIds struct {
 	sync.RWMutex
-	index map[uint16]Token
+	index map[uint16]tokenCompletor
 }
 
 const (
@@ -44,11 +45,14 @@ func (mids *messageIds) cleanUp() {
 			t.err = fmt.Errorf("Connection lost before Subscribe completed")
 		case *UnsubscribeToken:
 			t.err = fmt.Errorf("Connection lost before Unsubscribe completed")
+		case nil:
+			continue
 		}
 		token.flowComplete()
 	}
-	mids.index = make(map[uint16]Token)
+	mids.index = make(map[uint16]tokenCompletor)
 	mids.Unlock()
+	DEBUG.Println(MID, "cleaned up")
 }
 
 func (mids *messageIds) freeID(id uint16) {
@@ -57,7 +61,19 @@ func (mids *messageIds) freeID(id uint16) {
 	mids.Unlock()
 }
 
-func (mids *messageIds) getID(t Token) uint16 {
+func (mids *messageIds) claimID(token tokenCompletor, id uint16) {
+	mids.Lock()
+	defer mids.Unlock()
+	if _, ok := mids.index[id]; !ok {
+		mids.index[id] = token
+	} else {
+		old := mids.index[id]
+		old.flowComplete()
+		mids.index[id] = token
+	}
+}
+
+func (mids *messageIds) getID(t tokenCompletor) uint16 {
 	mids.Lock()
 	defer mids.Unlock()
 	for i := midMin; i < midMax; i++ {
@@ -69,11 +85,33 @@ func (mids *messageIds) getID(t Token) uint16 {
 	return 0
 }
 
-func (mids *messageIds) getToken(id uint16) Token {
+func (mids *messageIds) getToken(id uint16) tokenCompletor {
 	mids.RLock()
 	defer mids.RUnlock()
 	if token, ok := mids.index[id]; ok {
 		return token
 	}
+	return &DummyToken{id: id}
+}
+
+type DummyToken struct {
+	id uint16
+}
+
+func (d *DummyToken) Wait() bool {
+	return true
+}
+
+func (d *DummyToken) WaitTimeout(t time.Duration) bool {
+	return true
+}
+
+func (d *DummyToken) flowComplete() {
+	ERROR.Printf("A lookup for token %d returned nil\n", d.id)
+}
+
+func (d *DummyToken) Error() error {
 	return nil
 }
+
+func (d *DummyToken) setError(e error) {}

vendor/github.com/eclipse/paho.mqtt.golang/net.go

@@ -19,11 +19,15 @@ import (
 	"errors"
 	"fmt"
 	"net"
+	"net/http"
 	"net/url"
+	"os"
 	"reflect"
+	"sync/atomic"
 	"time"
 
 	"github.com/eclipse/paho.mqtt.golang/packets"
+	"golang.org/x/net/proxy"
 	"golang.org/x/net/websocket"
 )
 
@@ -34,10 +38,14 @@ func signalError(c chan<- error, err error) {
 	}
 }
 
-func openConnection(uri *url.URL, tlsc *tls.Config, timeout time.Duration) (net.Conn, error) {
+func openConnection(uri *url.URL, tlsc *tls.Config, timeout time.Duration, headers http.Header) (net.Conn, error) {
 	switch uri.Scheme {
 	case "ws":
-		conn, err := websocket.Dial(uri.String(), "mqtt", fmt.Sprintf("http://%s", uri.Host))
+		config, _ := websocket.NewConfig(uri.String(), fmt.Sprintf("http://%s", uri.Host))
+		config.Protocol = []string{"mqtt"}
+		config.Header = headers
+		config.Dialer = &net.Dialer{Timeout: timeout}
+		conn, err := websocket.DialConfig(config)
 		if err != nil {
 			return nil, err
 		}
@@ -47,6 +55,8 @@ func openConnection(uri *url.URL, tlsc *tls.Config, timeout time.Duration) (net.
 		config, _ := websocket.NewConfig(uri.String(), fmt.Sprintf("https://%s", uri.Host))
 		config.Protocol = []string{"mqtt"}
 		config.TlsConfig = tlsc
+		config.Header = headers
+		config.Dialer = &net.Dialer{Timeout: timeout}
 		conn, err := websocket.DialConfig(config)
 		if err != nil {
 			return nil, err
@@ -54,7 +64,23 @@ func openConnection(uri *url.URL, tlsc *tls.Config, timeout time.Duration) (net.
 		conn.PayloadType = websocket.BinaryFrame
 		return conn, err
 	case "tcp":
-		conn, err := net.DialTimeout("tcp", uri.Host, timeout)
+		allProxy := os.Getenv("all_proxy")
+		if len(allProxy) == 0 {
+			conn, err := net.DialTimeout("tcp", uri.Host, timeout)
+			if err != nil {
+				return nil, err
+			}
+			return conn, nil
+		}
+		proxyDialer := proxy.FromEnvironment()
+
+		conn, err := proxyDialer.Dial("tcp", uri.Host)
+		if err != nil {
+			return nil, err
+		}
+		return conn, nil
+	case "unix":
+		conn, err := net.DialTimeout("unix", uri.Host, timeout)
 		if err != nil {
 			return nil, err
 		}
@@ -64,11 +90,30 @@ func openConnection(uri *url.URL, tlsc *tls.Config, timeout time.Duration) (net.
 	case "tls":
 		fallthrough
 	case "tcps":
-		conn, err := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", uri.Host, tlsc)
+		allProxy := os.Getenv("all_proxy")
+		if len(allProxy) == 0 {
+			conn, err := tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", uri.Host, tlsc)
+			if err != nil {
+				return nil, err
+			}
+			return conn, nil
+		}
+		proxyDialer := proxy.FromEnvironment()
+
+		conn, err := proxyDialer.Dial("tcp", uri.Host)
 		if err != nil {
 			return nil, err
 		}
-		return conn, nil
+
+		tlsConn := tls.Client(conn, tlsc)
+
+		err = tlsConn.Handshake()
+		if err != nil {
+			conn.Close()
+			return nil, err
+		}
+
+		return tlsConn, nil
 	}
 	return nil, errors.New("Unknown protocol")
 }
@@ -92,7 +137,7 @@ func incoming(c *client) {
 		case c.ibound <- cp:
 			// Notify keepalive logic that we recently received a packet
 			if c.options.KeepAlive != 0 {
-				c.packetResp <- struct{}{}
+				atomic.StoreInt64(&c.lastReceived, time.Now().Unix())
 			}
 		case <-c.stop:
 			// This avoids a deadlock should a message arrive while shutting down.
@@ -136,6 +181,7 @@ func outgoing(c *client) {
 
 			if err := msg.Write(c.conn); err != nil {
 				ERROR.Println(NET, "outgoing stopped with error", err)
+				pub.t.setError(err)
 				signalError(c.errors, err)
 				return
 			}
@@ -160,6 +206,9 @@ func outgoing(c *client) {
 			DEBUG.Println(NET, "obound priority msg to write, type", reflect.TypeOf(msg.p))
 			if err := msg.p.Write(c.conn); err != nil {
 				ERROR.Println(NET, "outgoing stopped with error", err)
+				if msg.t != nil {
+					msg.t.setError(err)
+				}
 				signalError(c.errors, err)
 				return
 			}
@@ -172,11 +221,7 @@ func outgoing(c *client) {
 		}
 		// Reset ping timer after sending control packet.
 		if c.options.KeepAlive != 0 {
-			select {
-			case c.keepaliveReset <- struct{}{}:
-			default:
-				DEBUG.Println(NET, "couldn't send keepalive signal in outbound as channel full")
-			}
+			atomic.StoreInt64(&c.lastSent, time.Now().Unix())
 		}
 	}
 }
@@ -199,20 +244,22 @@ func alllogic(c *client) {
 			switch m := msg.(type) {
 			case *packets.PingrespPacket:
 				DEBUG.Println(NET, "received pingresp")
-				c.pingResp <- struct{}{}
+				atomic.StoreInt32(&c.pingOutstanding, 0)
 			case *packets.SubackPacket:
 				DEBUG.Println(NET, "received suback, id:", m.MessageID)
-				token := c.getToken(m.MessageID).(*SubscribeToken)
-				DEBUG.Println(NET, "granted qoss", m.ReturnCodes)
-				for i, qos := range m.ReturnCodes {
-					token.subResult[token.subs[i]] = qos
+				token := c.getToken(m.MessageID)
+				switch t := token.(type) {
+				case *SubscribeToken:
+					DEBUG.Println(NET, "granted qoss", m.ReturnCodes)
+					for i, qos := range m.ReturnCodes {
+						t.subResult[t.subs[i]] = qos
+					}
 				}
 				token.flowComplete()
 				c.freeID(m.MessageID)
 			case *packets.UnsubackPacket:
 				DEBUG.Println(NET, "received unsuback, id:", m.MessageID)
-				token := c.getToken(m.MessageID).(*UnsubscribeToken)
-				token.flowComplete()
+				c.getToken(m.MessageID).flowComplete()
 				c.freeID(m.MessageID)
 			case *packets.PublishPacket:
 				DEBUG.Println(NET, "received publish, msgId:", m.MessageID)
@@ -221,21 +268,14 @@ func alllogic(c *client) {
 				case 2:
 					c.incomingPubChan <- m
 					DEBUG.Println(NET, "done putting msg on incomingPubChan")
-					pr := packets.NewControlPacket(packets.Pubrec).(*packets.PubrecPacket)
-					pr.MessageID = m.MessageID
-					DEBUG.Println(NET, "putting pubrec msg on obound")
-					c.oboundP <- &PacketAndToken{p: pr, t: nil}
-					DEBUG.Println(NET, "done putting pubrec msg on obound")
 				case 1:
 					c.incomingPubChan <- m
 					DEBUG.Println(NET, "done putting msg on incomingPubChan")
-					pa := packets.NewControlPacket(packets.Puback).(*packets.PubackPacket)
-					pa.MessageID = m.MessageID
-					DEBUG.Println(NET, "putting puback msg on obound")
-					c.oboundP <- &PacketAndToken{p: pa, t: nil}
-					DEBUG.Println(NET, "done putting puback msg on obound")
 				case 0:
-					c.incomingPubChan <- m
+					select {
+					case c.incomingPubChan <- m:
+					case <-c.stop:
+					}
 					DEBUG.Println(NET, "done putting msg on incomingPubChan")
 				}
 			case *packets.PubackPacket:
@@ -250,15 +290,16 @@ func alllogic(c *client) {
 				prel.MessageID = m.MessageID
 				select {
 				case c.oboundP <- &PacketAndToken{p: prel, t: nil}:
-				case <-time.After(time.Second):
+				case <-c.stop:
 				}
 			case *packets.PubrelPacket:
 				DEBUG.Println(NET, "received pubrel, id:", m.MessageID)
 				pc := packets.NewControlPacket(packets.Pubcomp).(*packets.PubcompPacket)
 				pc.MessageID = m.MessageID
+				persistOutbound(c.persist, pc)
 				select {
 				case c.oboundP <- &PacketAndToken{p: pc, t: nil}:
-				case <-time.After(time.Second):
+				case <-c.stop:
 				}
 			case *packets.PubcompPacket:
 				DEBUG.Println(NET, "received pubcomp, id:", m.MessageID)
@@ -272,14 +313,43 @@ func alllogic(c *client) {
 	}
 }
 
+func (c *client) ackFunc(packet *packets.PublishPacket) func() {
+	return func() {
+		switch packet.Qos {
+		case 2:
+			pr := packets.NewControlPacket(packets.Pubrec).(*packets.PubrecPacket)
+			pr.MessageID = packet.MessageID
+			DEBUG.Println(NET, "putting pubrec msg on obound")
+			select {
+			case c.oboundP <- &PacketAndToken{p: pr, t: nil}:
+			case <-c.stop:
+			}
+			DEBUG.Println(NET, "done putting pubrec msg on obound")
+		case 1:
+			pa := packets.NewControlPacket(packets.Puback).(*packets.PubackPacket)
+			pa.MessageID = packet.MessageID
+			DEBUG.Println(NET, "putting puback msg on obound")
+			persistOutbound(c.persist, pa)
+			select {
+			case c.oboundP <- &PacketAndToken{p: pa, t: nil}:
+			case <-c.stop:
+			}
+			DEBUG.Println(NET, "done putting puback msg on obound")
+		case 0:
+			// do nothing, since there is no need to send an ack packet back
+		}
+	}
+}
+
 func errorWatch(c *client) {
+	defer c.workers.Done()
 	select {
 	case <-c.stop:
 		WARN.Println(NET, "errorWatch stopped")
 		return
 	case err := <-c.errors:
 		ERROR.Println(NET, "error triggered, stopping")
-		c.internalConnLost(err)
+		go c.internalConnLost(err)
 		return
 	}
 }

vendor/github.com/eclipse/paho.mqtt.golang/options.go

@@ -12,14 +12,22 @@
  *    Mike Robertson
  */
 
+// Portions copyright © 2018 TIBCO Software Inc.
+
 package mqtt
 
 import (
 	"crypto/tls"
+	"net/http"
 	"net/url"
+	"strings"
 	"time"
 )
 
+// CredentialsProvider allows the username and password to be updated
+// before reconnecting. It should return the current username and password.
+type CredentialsProvider func() (username string, password string)
+
 // MessageHandler is a callback type which can be set to be
 // executed upon the arrival of messages published to topics
 // to which the client is subscribed.
@@ -42,6 +50,7 @@ type ClientOptions struct {
 	ClientID                string
 	Username                string
 	Password                string
+	CredentialsProvider     CredentialsProvider
 	CleanSession            bool
 	Order                   bool
 	WillEnabled             bool
@@ -51,18 +60,20 @@ type ClientOptions struct {
 	WillRetained            bool
 	ProtocolVersion         uint
 	protocolVersionExplicit bool
-	TLSConfig               tls.Config
-	KeepAlive               time.Duration
+	TLSConfig               *tls.Config
+	KeepAlive               int64
 	PingTimeout             time.Duration
 	ConnectTimeout          time.Duration
 	MaxReconnectInterval    time.Duration
 	AutoReconnect           bool
 	Store                   Store
-	DefaultPublishHander    MessageHandler
+	DefaultPublishHandler   MessageHandler
 	OnConnect               OnConnectHandler
 	OnConnectionLost        ConnectionLostHandler
 	WriteTimeout            time.Duration
 	MessageChannelDepth     uint
+	ResumeSubs              bool
+	HTTPHeaders             http.Header
 }
 
 // NewClientOptions will create a new ClientClientOptions type with some
@@ -89,8 +100,7 @@ func NewClientOptions() *ClientOptions {
 		WillRetained:            false,
 		ProtocolVersion:         0,
 		protocolVersionExplicit: false,
-		TLSConfig:               tls.Config{},
-		KeepAlive:               30 * time.Second,
+		KeepAlive:               30,
 		PingTimeout:             10 * time.Second,
 		ConnectTimeout:          30 * time.Second,
 		MaxReconnectInterval:    10 * time.Minute,
@@ -100,6 +110,8 @@ func NewClientOptions() *ClientOptions {
 		OnConnectionLost:        DefaultConnectionLostHandler,
 		WriteTimeout:            0, // 0 represents timeout disabled
 		MessageChannelDepth:     100,
+		ResumeSubs:              false,
+		HTTPHeaders:             make(map[string][]string),
 	}
 	return o
 }
@@ -108,11 +120,30 @@ func NewClientOptions() *ClientOptions {
 // scheme://host:port
 // Where "scheme" is one of "tcp", "ssl", or "ws", "host" is the ip-address (or hostname)
 // and "port" is the port on which the broker is accepting connections.
+//
+// Default values for hostname is "127.0.0.1", for schema is "tcp://".
+//
+// An example broker URI would look like: tcp://foobar.com:1883
 func (o *ClientOptions) AddBroker(server string) *ClientOptions {
-	brokerURI, err := url.Parse(server)
-	if err == nil {
-		o.Servers = append(o.Servers, brokerURI)
+	if len(server) > 0 && server[0] == ':' {
+		server = "127.0.0.1" + server
 	}
+	if !strings.Contains(server, "://") {
+		server = "tcp://" + server
+	}
+	brokerURI, err := url.Parse(server)
+	if err != nil {
+		ERROR.Println(CLI, "Failed to parse %q broker address: %s", server, err)
+		return o
+	}
+	o.Servers = append(o.Servers, brokerURI)
+	return o
+}
+
+// SetResumeSubs will enable resuming of stored (un)subscribe messages when connecting
+// but not reconnecting if CleanSession is false. Otherwise these messages are discarded.
+func (o *ClientOptions) SetResumeSubs(resume bool) *ClientOptions {
+	o.ResumeSubs = resume
 	return o
 }
 
@@ -140,6 +171,15 @@ func (o *ClientOptions) SetPassword(p string) *ClientOptions {
 	return o
 }
 
+// SetCredentialsProvider will set a method to be called by this client when
+// connecting to the MQTT broker that provide the current username and password.
+// Note: without the use of SSL/TLS, this information will be sent
+// in plaintext accross the wire.
+func (o *ClientOptions) SetCredentialsProvider(p CredentialsProvider) *ClientOptions {
+	o.CredentialsProvider = p
+	return o
+}
+
 // SetCleanSession will set the "clean session" flag in the connect message
 // when this client connects to an MQTT broker. By setting this flag, you are
 // indicating that no messages saved by the broker for this client should be
@@ -164,7 +204,7 @@ func (o *ClientOptions) SetOrderMatters(order bool) *ClientOptions {
 // to an MQTT broker. Please read the official Go documentation for more
 // information.
 func (o *ClientOptions) SetTLSConfig(t *tls.Config) *ClientOptions {
-	o.TLSConfig = *t
+	o.TLSConfig = t
 	return o
 }
 
@@ -182,7 +222,7 @@ func (o *ClientOptions) SetStore(s Store) *ClientOptions {
 // allow the client to know that a connection has not been lost with the
 // server.
 func (o *ClientOptions) SetKeepAlive(k time.Duration) *ClientOptions {
-	o.KeepAlive = k
+	o.KeepAlive = int64(k / time.Second)
 	return o
 }
 
@@ -197,7 +237,7 @@ func (o *ClientOptions) SetPingTimeout(k time.Duration) *ClientOptions {
 // SetProtocolVersion sets the MQTT version to be used to connect to the
 // broker. Legitimate values are currently 3 - MQTT 3.1 or 4 - MQTT 3.1.1
 func (o *ClientOptions) SetProtocolVersion(pv uint) *ClientOptions {
-	if pv >= 3 && pv <= 4 {
+	if (pv >= 3 && pv <= 4) || (pv > 0x80) {
 		o.ProtocolVersion = pv
 		o.protocolVersionExplicit = true
 	}
@@ -235,7 +275,7 @@ func (o *ClientOptions) SetBinaryWill(topic string, payload []byte, qos byte, re
 // SetDefaultPublishHandler sets the MessageHandler that will be called when a message
 // is received that does not match any known subscriptions.
 func (o *ClientOptions) SetDefaultPublishHandler(defaultHandler MessageHandler) *ClientOptions {
-	o.DefaultPublishHander = defaultHandler
+	o.DefaultPublishHandler = defaultHandler
 	return o
 }
 
@@ -291,3 +331,10 @@ func (o *ClientOptions) SetMessageChannelDepth(s uint) *ClientOptions {
 	o.MessageChannelDepth = s
 	return o
 }
+
+// SetHTTPHeaders sets the additional HTTP headers that will be sent in the WebSocket
+// opening handshake.
+func (o *ClientOptions) SetHTTPHeaders(h http.Header) *ClientOptions {
+	o.HTTPHeaders = h
+	return o
+}

vendor/github.com/eclipse/paho.mqtt.golang/options_reader.go

@@ -16,6 +16,7 @@ package mqtt
 
 import (
 	"crypto/tls"
+	"net/http"
 	"net/url"
 	"time"
 )
@@ -25,6 +26,7 @@ type ClientOptionsReader struct {
 	options *ClientOptions
 }
 
+//Servers returns a slice of the servers defined in the clientoptions
 func (r *ClientOptionsReader) Servers() []*url.URL {
 	s := make([]*url.URL, len(r.options.Servers))
 
@@ -36,21 +38,31 @@ func (r *ClientOptionsReader) Servers() []*url.URL {
 	return s
 }
 
+//ResumeSubs returns true if resuming stored (un)sub is enabled
+func (r *ClientOptionsReader) ResumeSubs() bool {
+	s := r.options.ResumeSubs
+	return s
+}
+
+//ClientID returns the set client id
 func (r *ClientOptionsReader) ClientID() string {
 	s := r.options.ClientID
 	return s
 }
 
+//Username returns the set username
 func (r *ClientOptionsReader) Username() string {
 	s := r.options.Username
 	return s
 }
 
+//Password returns the set password
 func (r *ClientOptionsReader) Password() string {
 	s := r.options.Password
 	return s
 }
 
+//CleanSession returns whether Cleansession is set
 func (r *ClientOptionsReader) CleanSession() bool {
 	s := r.options.CleanSession
 	return s
@@ -91,13 +103,13 @@ func (r *ClientOptionsReader) ProtocolVersion() uint {
 	return s
 }
 
-func (r *ClientOptionsReader) TLSConfig() tls.Config {
+func (r *ClientOptionsReader) TLSConfig() *tls.Config {
 	s := r.options.TLSConfig
 	return s
 }
 
 func (r *ClientOptionsReader) KeepAlive() time.Duration {
-	s := r.options.KeepAlive
+	s := time.Duration(r.options.KeepAlive * int64(time.Second))
 	return s
 }
 
@@ -130,3 +142,8 @@ func (r *ClientOptionsReader) MessageChannelDepth() uint {
 	s := r.options.MessageChannelDepth
 	return s
 }
+
+func (r *ClientOptionsReader) HTTPHeaders() http.Header {
+	h := r.options.HTTPHeaders
+	return h
+}

vendor/github.com/eclipse/paho.mqtt.golang/ping.go

@@ -16,87 +16,51 @@ package mqtt
 
 import (
 	"errors"
+	"sync/atomic"
 	"time"
 
 	"github.com/eclipse/paho.mqtt.golang/packets"
 )
 
 func keepalive(c *client) {
+	defer c.workers.Done()
 	DEBUG.Println(PNG, "keepalive starting")
+	var checkInterval int64
+	var pingSent time.Time
 
-	receiveInterval := c.options.KeepAlive + (1 * time.Second)
-	pingTimer := timer{Timer: time.NewTimer(c.options.KeepAlive)}
-	receiveTimer := timer{Timer: time.NewTimer(receiveInterval)}
-	pingRespTimer := timer{Timer: time.NewTimer(c.options.PingTimeout)}
+	if c.options.KeepAlive > 10 {
+		checkInterval = 5
+	} else {
+		checkInterval = c.options.KeepAlive / 2
+	}
 
-	pingRespTimer.Stop()
+	intervalTicker := time.NewTicker(time.Duration(checkInterval * int64(time.Second)))
+	defer intervalTicker.Stop()
 
 	for {
 		select {
 		case <-c.stop:
 			DEBUG.Println(PNG, "keepalive stopped")
-			c.workers.Done()
-			return
-		case <-pingTimer.C:
-			sendPing(&pingTimer, &pingRespTimer, c)
-		case <-c.keepaliveReset:
-			DEBUG.Println(NET, "resetting ping timer")
-			pingTimer.Reset(c.options.KeepAlive)
-		case <-c.pingResp:
-			DEBUG.Println(NET, "resetting ping timeout timer")
-			pingRespTimer.Stop()
-			pingTimer.Reset(c.options.KeepAlive)
-			receiveTimer.Reset(receiveInterval)
-		case <-c.packetResp:
-			DEBUG.Println(NET, "resetting receive timer")
-			receiveTimer.Reset(receiveInterval)
-		case <-receiveTimer.C:
-			receiveTimer.SetRead(true)
-			receiveTimer.Reset(receiveInterval)
-			sendPing(&pingTimer, &pingRespTimer, c)
-		case <-pingRespTimer.C:
-			pingRespTimer.SetRead(true)
-			CRITICAL.Println(PNG, "pingresp not received, disconnecting")
-			c.workers.Done()
-			c.internalConnLost(errors.New("pingresp not received, disconnecting"))
-			pingTimer.Stop()
 			return
+		case <-intervalTicker.C:
+			DEBUG.Println(PNG, "ping check", time.Now().Unix()-atomic.LoadInt64(&c.lastSent))
+			if time.Now().Unix()-atomic.LoadInt64(&c.lastSent) >= c.options.KeepAlive || time.Now().Unix()-atomic.LoadInt64(&c.lastReceived) >= c.options.KeepAlive {
+				if atomic.LoadInt32(&c.pingOutstanding) == 0 {
+					DEBUG.Println(PNG, "keepalive sending ping")
+					ping := packets.NewControlPacket(packets.Pingreq).(*packets.PingreqPacket)
+					//We don't want to wait behind large messages being sent, the Write call
+					//will block until it it able to send the packet.
+					atomic.StoreInt32(&c.pingOutstanding, 1)
+					ping.Write(c.conn)
+					atomic.StoreInt64(&c.lastSent, time.Now().Unix())
+					pingSent = time.Now()
+				}
+			}
+			if atomic.LoadInt32(&c.pingOutstanding) > 0 && time.Now().Sub(pingSent) >= c.options.PingTimeout {
+				CRITICAL.Println(PNG, "pingresp not received, disconnecting")
+				c.errors <- errors.New("pingresp not received, disconnecting")
+				return
+			}
 		}
 	}
 }
-
-type timer struct {
-	*time.Timer
-	readFrom bool
-}
-
-func (t *timer) SetRead(v bool) {
-	t.readFrom = v
-}
-
-func (t *timer) Stop() bool {
-	defer t.SetRead(true)
-
-	if !t.Timer.Stop() && !t.readFrom {
-		<-t.C
-		return false
-	}
-	return true
-}
-
-func (t *timer) Reset(d time.Duration) bool {
-	defer t.SetRead(false)
-	t.Stop()
-	return t.Timer.Reset(d)
-}
-
-func sendPing(pt *timer, rt *timer, c *client) {
-	pt.SetRead(true)
-	DEBUG.Println(PNG, "keepalive sending ping")
-	ping := packets.NewControlPacket(packets.Pingreq).(*packets.PingreqPacket)
-	//We don't want to wait behind large messages being sent, the Write call
-	//will block until it it able to send the packet.
-	ping.Write(c.conn)
-
-	rt.Reset(c.options.PingTimeout)
-}

vendor/github.com/eclipse/paho.mqtt.golang/router.go

@@ -57,12 +57,23 @@ func match(route []string, topic []string) bool {
 	if (route[0] == "+") || (route[0] == topic[0]) {
 		return match(route[1:], topic[1:])
 	}
-
 	return false
 }
 
 func routeIncludesTopic(route, topic string) bool {
-	return match(strings.Split(route, "/"), strings.Split(topic, "/"))
+	return match(routeSplit(route), strings.Split(topic, "/"))
+}
+
+// removes $share and sharename when splitting the route to allow
+// shared subscription routes to correctly match the topic
+func routeSplit(route string) []string {
+	var result []string
+	if strings.HasPrefix(route, "$share") {
+		result = strings.Split(route, "/")[2:]
+	} else {
+		result = strings.Split(route, "/")
+	}
+	return result
 }
 
 // match takes the topic string of the published message and does a basic compare to the
@@ -135,24 +146,39 @@ func (r *router) matchAndDispatch(messages <-chan *packets.PublishPacket, order
 			case message := <-messages:
 				sent := false
 				r.RLock()
+				m := messageFromPublish(message, client.ackFunc(message))
+				handlers := []MessageHandler{}
 				for e := r.routes.Front(); e != nil; e = e.Next() {
 					if e.Value.(*route).match(message.TopicName) {
 						if order {
-							e.Value.(*route).callback(client, messageFromPublish(message))
+							handlers = append(handlers, e.Value.(*route).callback)
 						} else {
-							go e.Value.(*route).callback(client, messageFromPublish(message))
+							hd := e.Value.(*route).callback
+							go func() {
+								hd(client, m)
+								m.Ack()
+							}()
 						}
 						sent = true
 					}
 				}
 				if !sent && r.defaultHandler != nil {
 					if order {
-						r.defaultHandler(client, messageFromPublish(message))
+						handlers = append(handlers, r.defaultHandler)
 					} else {
-						go r.defaultHandler(client, messageFromPublish(message))
+						go func() {
+							r.defaultHandler(client, m)
+							m.Ack()
+						}()
 					}
 				}
 				r.RUnlock()
+				for _, handler := range handlers {
+					func() {
+						handler(client, m)
+						m.Ack()
+					}()
+				}
 			case <-r.stop:
 				return
 			}

vendor/github.com/eclipse/paho.mqtt.golang/store.go

@@ -50,6 +50,16 @@ func mIDFromKey(key string) uint16 {
 	return uint16(i)
 }
 
+// Return true if key prefix is outbound
+func isKeyOutbound(key string) bool {
+	return key[:2] == outboundPrefix
+}
+
+// Return true if key prefix is inbound
+func isKeyInbound(key string) bool {
+	return key[:2] == inboundPrefix
+}
+
 // Return a string of the form "i.[id]"
 func inboundKeyFromMID(id uint16) string {
 	return fmt.Sprintf("%s%d", inboundPrefix, id)

vendor/github.com/eclipse/paho.mqtt.golang/token.go

@@ -19,24 +19,33 @@ import (
 	"github.com/eclipse/paho.mqtt.golang/packets"
 )
 
-//PacketAndToken is a struct that contains both a ControlPacket and a
-//Token. This struct is passed via channels between the client interface
-//code and the underlying code responsible for sending and receiving
-//MQTT messages.
+// PacketAndToken is a struct that contains both a ControlPacket and a
+// Token. This struct is passed via channels between the client interface
+// code and the underlying code responsible for sending and receiving
+// MQTT messages.
 type PacketAndToken struct {
 	p packets.ControlPacket
-	t Token
+	t tokenCompletor
 }
 
-//Token defines the interface for the tokens used to indicate when
-//actions have completed.
+// Token defines the interface for the tokens used to indicate when
+// actions have completed.
 type Token interface {
 	Wait() bool
 	WaitTimeout(time.Duration) bool
-	flowComplete()
 	Error() error
 }
 
+type TokenErrorSetter interface {
+	setError(error)
+}
+
+type tokenCompletor interface {
+	Token
+	TokenErrorSetter
+	flowComplete()
+}
+
 type baseToken struct {
 	m        sync.RWMutex
 	complete chan struct{}
@@ -56,25 +65,34 @@ func (b *baseToken) Wait() bool {
 	return b.ready
 }
 
-// WaitTimeout takes a time in ms to wait for the flow associated with the
+// WaitTimeout takes a time.Duration to wait for the flow associated with the
 // Token to complete, returns true if it returned before the timeout or
 // returns false if the timeout occurred. In the case of a timeout the Token
 // does not have an error set in case the caller wishes to wait again
 func (b *baseToken) WaitTimeout(d time.Duration) bool {
 	b.m.Lock()
 	defer b.m.Unlock()
+
 	if !b.ready {
+		timer := time.NewTimer(d)
 		select {
 		case <-b.complete:
 			b.ready = true
-		case <-time.After(d):
+			if !timer.Stop() {
+				<-timer.C
+			}
+		case <-timer.C:
 		}
 	}
 	return b.ready
 }
 
 func (b *baseToken) flowComplete() {
-	close(b.complete)
+	select {
+	case <-b.complete:
+	default:
+		close(b.complete)
+	}
 }
 
 func (b *baseToken) Error() error {
@@ -83,7 +101,12 @@ func (b *baseToken) Error() error {
 	return b.err
 }
 
-func newToken(tType byte) Token {
+func (b *baseToken) setError(e error) {
+	b.err = e
+	b.flowComplete()
+}
+
+func newToken(tType byte) tokenCompletor {
 	switch tType {
 	case packets.Connect:
 		return &ConnectToken{baseToken: baseToken{complete: make(chan struct{})}}
@@ -99,59 +122,68 @@ func newToken(tType byte) Token {
 	return nil
 }
 
-//ConnectToken is an extension of Token containing the extra fields
-//required to provide information about calls to Connect()
+// ConnectToken is an extension of Token containing the extra fields
+// required to provide information about calls to Connect()
 type ConnectToken struct {
 	baseToken
-	returnCode byte
+	returnCode     byte
+	sessionPresent bool
 }
 
-//ReturnCode returns the acknowlegement code in the connack sent
-//in response to a Connect()
+// ReturnCode returns the acknowlegement code in the connack sent
+// in response to a Connect()
 func (c *ConnectToken) ReturnCode() byte {
 	c.m.RLock()
 	defer c.m.RUnlock()
 	return c.returnCode
 }
 
-//PublishToken is an extension of Token containing the extra fields
-//required to provide information about calls to Publish()
+// SessionPresent returns a bool representing the value of the
+// session present field in the connack sent in response to a Connect()
+func (c *ConnectToken) SessionPresent() bool {
+	c.m.RLock()
+	defer c.m.RUnlock()
+	return c.sessionPresent
+}
+
+// PublishToken is an extension of Token containing the extra fields
+// required to provide information about calls to Publish()
 type PublishToken struct {
 	baseToken
 	messageID uint16
 }
 
-//MessageID returns the MQTT message ID that was assigned to the
-//Publish packet when it was sent to the broker
+// MessageID returns the MQTT message ID that was assigned to the
+// Publish packet when it was sent to the broker
 func (p *PublishToken) MessageID() uint16 {
 	return p.messageID
 }
 
-//SubscribeToken is an extension of Token containing the extra fields
-//required to provide information about calls to Subscribe()
+// SubscribeToken is an extension of Token containing the extra fields
+// required to provide information about calls to Subscribe()
 type SubscribeToken struct {
 	baseToken
 	subs      []string
 	subResult map[string]byte
 }
 
-//Result returns a map of topics that were subscribed to along with
-//the matching return code from the broker. This is either the Qos
-//value of the subscription or an error code.
+// Result returns a map of topics that were subscribed to along with
+// the matching return code from the broker. This is either the Qos
+// value of the subscription or an error code.
 func (s *SubscribeToken) Result() map[string]byte {
 	s.m.RLock()
 	defer s.m.RUnlock()
 	return s.subResult
 }
 
-//UnsubscribeToken is an extension of Token containing the extra fields
-//required to provide information about calls to Unsubscribe()
+// UnsubscribeToken is an extension of Token containing the extra fields
+// required to provide information about calls to Unsubscribe()
 type UnsubscribeToken struct {
 	baseToken
 }
 
-//DisconnectToken is an extension of Token containing the extra fields
-//required to provide information about calls to Disconnect()
+// DisconnectToken is an extension of Token containing the extra fields
+// required to provide information about calls to Disconnect()
 type DisconnectToken struct {
 	baseToken
 }

vendor/github.com/eclipse/paho.mqtt.golang/trace.go

@@ -14,23 +14,27 @@
 
 package mqtt
 
-import (
-	"io/ioutil"
-	"log"
+type (
+	// Logger interface allows implementations to provide to this package any
+	// object that implements the methods defined in it.
+	Logger interface {
+		Println(v ...interface{})
+		Printf(format string, v ...interface{})
+	}
+
+	// NOOPLogger implements the logger that does not perform any operation
+	// by default. This allows us to efficiently discard the unwanted messages.
+	NOOPLogger struct{}
 )
 
+func (NOOPLogger) Println(v ...interface{}) {}
+func (NOOPLogger) Printf(format string, v ...interface{}) {}
+
 // Internal levels of library output that are initialised to not print
 // anything but can be overridden by programmer
 var (
-	ERROR    *log.Logger
-	CRITICAL *log.Logger
-	WARN     *log.Logger
-	DEBUG    *log.Logger
+	ERROR    Logger = NOOPLogger{}
+	CRITICAL Logger = NOOPLogger{}
+	WARN     Logger = NOOPLogger{}
+	DEBUG    Logger = NOOPLogger{}
 )
-
-func init() {
-	ERROR = log.New(ioutil.Discard, "", 0)
-	CRITICAL = log.New(ioutil.Discard, "", 0)
-	WARN = log.New(ioutil.Discard, "", 0)
-	DEBUG = log.New(ioutil.Discard, "", 0)
-}

vendor/github.com/minio/parquet-go/column.go

@@ -71,7 +71,9 @@ func getColumns(
 			valueType:      meta.GetType(),
 		}
 
-		nameIndexMap[columnName] = colIndex
+		// First element of []*parquet.SchemaElement from parquet file metadata is 'schema'
+		// which is always skipped, hence colIndex + 1 is valid.
+		nameIndexMap[columnName] = colIndex + 1
 	}
 
 	for name := range nameColumnMap {

vendor/github.com/minio/parquet-go/decode.go

@@ -194,6 +194,8 @@ func readDoubles(reader *bytes.Reader, count uint64) (result []float64, err erro
 
 func readByteArrays(reader *bytes.Reader, count uint64) (result [][]byte, err error) {
 	buf := make([]byte, 4)
+	var length uint32
+	var data []byte
 
 	var i uint64
 	for i = 0; i < count; i++ {
@@ -201,9 +203,12 @@ func readByteArrays(reader *bytes.Reader, count uint64) (result [][]byte, err er
 			return nil, err
 		}
 
-		data := make([]byte, bytesToUint32(buf))
-		if _, err = reader.Read(data); err != nil {
-			return nil, err
+		length = bytesToUint32(buf)
+		data = make([]byte, length)
+		if length > 0 {
+			if _, err = reader.Read(data); err != nil {
+				return nil, err
+			}
 		}
 
 		result = append(result, data)

vendor/golang.org/x/net/internal/socks/client.go

@@ -0,0 +1,168 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package socks
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"strconv"
+	"time"
+)
+
+var (
+	noDeadline   = time.Time{}
+	aLongTimeAgo = time.Unix(1, 0)
+)
+
+func (d *Dialer) connect(ctx context.Context, c net.Conn, address string) (_ net.Addr, ctxErr error) {
+	host, port, err := splitHostPort(address)
+	if err != nil {
+		return nil, err
+	}
+	if deadline, ok := ctx.Deadline(); ok && !deadline.IsZero() {
+		c.SetDeadline(deadline)
+		defer c.SetDeadline(noDeadline)
+	}
+	if ctx != context.Background() {
+		errCh := make(chan error, 1)
+		done := make(chan struct{})
+		defer func() {
+			close(done)
+			if ctxErr == nil {
+				ctxErr = <-errCh
+			}
+		}()
+		go func() {
+			select {
+			case <-ctx.Done():
+				c.SetDeadline(aLongTimeAgo)
+				errCh <- ctx.Err()
+			case <-done:
+				errCh <- nil
+			}
+		}()
+	}
+
+	b := make([]byte, 0, 6+len(host)) // the size here is just an estimate
+	b = append(b, Version5)
+	if len(d.AuthMethods) == 0 || d.Authenticate == nil {
+		b = append(b, 1, byte(AuthMethodNotRequired))
+	} else {
+		ams := d.AuthMethods
+		if len(ams) > 255 {
+			return nil, errors.New("too many authentication methods")
+		}
+		b = append(b, byte(len(ams)))
+		for _, am := range ams {
+			b = append(b, byte(am))
+		}
+	}
+	if _, ctxErr = c.Write(b); ctxErr != nil {
+		return
+	}
+
+	if _, ctxErr = io.ReadFull(c, b[:2]); ctxErr != nil {
+		return
+	}
+	if b[0] != Version5 {
+		return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
+	}
+	am := AuthMethod(b[1])
+	if am == AuthMethodNoAcceptableMethods {
+		return nil, errors.New("no acceptable authentication methods")
+	}
+	if d.Authenticate != nil {
+		if ctxErr = d.Authenticate(ctx, c, am); ctxErr != nil {
+			return
+		}
+	}
+
+	b = b[:0]
+	b = append(b, Version5, byte(d.cmd), 0)
+	if ip := net.ParseIP(host); ip != nil {
+		if ip4 := ip.To4(); ip4 != nil {
+			b = append(b, AddrTypeIPv4)
+			b = append(b, ip4...)
+		} else if ip6 := ip.To16(); ip6 != nil {
+			b = append(b, AddrTypeIPv6)
+			b = append(b, ip6...)
+		} else {
+			return nil, errors.New("unknown address type")
+		}
+	} else {
+		if len(host) > 255 {
+			return nil, errors.New("FQDN too long")
+		}
+		b = append(b, AddrTypeFQDN)
+		b = append(b, byte(len(host)))
+		b = append(b, host...)
+	}
+	b = append(b, byte(port>>8), byte(port))
+	if _, ctxErr = c.Write(b); ctxErr != nil {
+		return
+	}
+
+	if _, ctxErr = io.ReadFull(c, b[:4]); ctxErr != nil {
+		return
+	}
+	if b[0] != Version5 {
+		return nil, errors.New("unexpected protocol version " + strconv.Itoa(int(b[0])))
+	}
+	if cmdErr := Reply(b[1]); cmdErr != StatusSucceeded {
+		return nil, errors.New("unknown error " + cmdErr.String())
+	}
+	if b[2] != 0 {
+		return nil, errors.New("non-zero reserved field")
+	}
+	l := 2
+	var a Addr
+	switch b[3] {
+	case AddrTypeIPv4:
+		l += net.IPv4len
+		a.IP = make(net.IP, net.IPv4len)
+	case AddrTypeIPv6:
+		l += net.IPv6len
+		a.IP = make(net.IP, net.IPv6len)
+	case AddrTypeFQDN:
+		if _, err := io.ReadFull(c, b[:1]); err != nil {
+			return nil, err
+		}
+		l += int(b[0])
+	default:
+		return nil, errors.New("unknown address type " + strconv.Itoa(int(b[3])))
+	}
+	if cap(b) < l {
+		b = make([]byte, l)
+	} else {
+		b = b[:l]
+	}
+	if _, ctxErr = io.ReadFull(c, b); ctxErr != nil {
+		return
+	}
+	if a.IP != nil {
+		copy(a.IP, b)
+	} else {
+		a.Name = string(b[:len(b)-2])
+	}
+	a.Port = int(b[len(b)-2])<<8 | int(b[len(b)-1])
+	return &a, nil
+}
+
+func splitHostPort(address string) (string, int, error) {
+	host, port, err := net.SplitHostPort(address)
+	if err != nil {
+		return "", 0, err
+	}
+	portnum, err := strconv.Atoi(port)
+	if err != nil {
+		return "", 0, err
+	}
+	if 1 > portnum || portnum > 0xffff {
+		return "", 0, errors.New("port number out of range " + port)
+	}
+	return host, portnum, nil
+}

vendor/golang.org/x/net/internal/socks/socks.go

@@ -0,0 +1,317 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package socks provides a SOCKS version 5 client implementation.
+//
+// SOCKS protocol version 5 is defined in RFC 1928.
+// Username/Password authentication for SOCKS version 5 is defined in
+// RFC 1929.
+package socks
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"strconv"
+)
+
+// A Command represents a SOCKS command.
+type Command int
+
+func (cmd Command) String() string {
+	switch cmd {
+	case CmdConnect:
+		return "socks connect"
+	case cmdBind:
+		return "socks bind"
+	default:
+		return "socks " + strconv.Itoa(int(cmd))
+	}
+}
+
+// An AuthMethod represents a SOCKS authentication method.
+type AuthMethod int
+
+// A Reply represents a SOCKS command reply code.
+type Reply int
+
+func (code Reply) String() string {
+	switch code {
+	case StatusSucceeded:
+		return "succeeded"
+	case 0x01:
+		return "general SOCKS server failure"
+	case 0x02:
+		return "connection not allowed by ruleset"
+	case 0x03:
+		return "network unreachable"
+	case 0x04:
+		return "host unreachable"
+	case 0x05:
+		return "connection refused"
+	case 0x06:
+		return "TTL expired"
+	case 0x07:
+		return "command not supported"
+	case 0x08:
+		return "address type not supported"
+	default:
+		return "unknown code: " + strconv.Itoa(int(code))
+	}
+}
+
+// Wire protocol constants.
+const (
+	Version5 = 0x05
+
+	AddrTypeIPv4 = 0x01
+	AddrTypeFQDN = 0x03
+	AddrTypeIPv6 = 0x04
+
+	CmdConnect Command = 0x01 // establishes an active-open forward proxy connection
+	cmdBind    Command = 0x02 // establishes a passive-open forward proxy connection
+
+	AuthMethodNotRequired         AuthMethod = 0x00 // no authentication required
+	AuthMethodUsernamePassword    AuthMethod = 0x02 // use username/password
+	AuthMethodNoAcceptableMethods AuthMethod = 0xff // no acceptable authentication methods
+
+	StatusSucceeded Reply = 0x00
+)
+
+// An Addr represents a SOCKS-specific address.
+// Either Name or IP is used exclusively.
+type Addr struct {
+	Name string // fully-qualified domain name
+	IP   net.IP
+	Port int
+}
+
+func (a *Addr) Network() string { return "socks" }
+
+func (a *Addr) String() string {
+	if a == nil {
+		return "<nil>"
+	}
+	port := strconv.Itoa(a.Port)
+	if a.IP == nil {
+		return net.JoinHostPort(a.Name, port)
+	}
+	return net.JoinHostPort(a.IP.String(), port)
+}
+
+// A Conn represents a forward proxy connection.
+type Conn struct {
+	net.Conn
+
+	boundAddr net.Addr
+}
+
+// BoundAddr returns the address assigned by the proxy server for
+// connecting to the command target address from the proxy server.
+func (c *Conn) BoundAddr() net.Addr {
+	if c == nil {
+		return nil
+	}
+	return c.boundAddr
+}
+
+// A Dialer holds SOCKS-specific options.
+type Dialer struct {
+	cmd          Command // either CmdConnect or cmdBind
+	proxyNetwork string  // network between a proxy server and a client
+	proxyAddress string  // proxy server address
+
+	// ProxyDial specifies the optional dial function for
+	// establishing the transport connection.
+	ProxyDial func(context.Context, string, string) (net.Conn, error)
+
+	// AuthMethods specifies the list of request authention
+	// methods.
+	// If empty, SOCKS client requests only AuthMethodNotRequired.
+	AuthMethods []AuthMethod
+
+	// Authenticate specifies the optional authentication
+	// function. It must be non-nil when AuthMethods is not empty.
+	// It must return an error when the authentication is failed.
+	Authenticate func(context.Context, io.ReadWriter, AuthMethod) error
+}
+
+// DialContext connects to the provided address on the provided
+// network.
+//
+// The returned error value may be a net.OpError. When the Op field of
+// net.OpError contains "socks", the Source field contains a proxy
+// server address and the Addr field contains a command target
+// address.
+//
+// See func Dial of the net package of standard library for a
+// description of the network and address parameters.
+func (d *Dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	if err := d.validateTarget(network, address); err != nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
+	}
+	if ctx == nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
+	}
+	var err error
+	var c net.Conn
+	if d.ProxyDial != nil {
+		c, err = d.ProxyDial(ctx, d.proxyNetwork, d.proxyAddress)
+	} else {
+		var dd net.Dialer
+		c, err = dd.DialContext(ctx, d.proxyNetwork, d.proxyAddress)
+	}
+	if err != nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
+	}
+	a, err := d.connect(ctx, c, address)
+	if err != nil {
+		c.Close()
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
+	}
+	return &Conn{Conn: c, boundAddr: a}, nil
+}
+
+// DialWithConn initiates a connection from SOCKS server to the target
+// network and address using the connection c that is already
+// connected to the SOCKS server.
+//
+// It returns the connection's local address assigned by the SOCKS
+// server.
+func (d *Dialer) DialWithConn(ctx context.Context, c net.Conn, network, address string) (net.Addr, error) {
+	if err := d.validateTarget(network, address); err != nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
+	}
+	if ctx == nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: errors.New("nil context")}
+	}
+	a, err := d.connect(ctx, c, address)
+	if err != nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
+	}
+	return a, nil
+}
+
+// Dial connects to the provided address on the provided network.
+//
+// Unlike DialContext, it returns a raw transport connection instead
+// of a forward proxy connection.
+//
+// Deprecated: Use DialContext or DialWithConn instead.
+func (d *Dialer) Dial(network, address string) (net.Conn, error) {
+	if err := d.validateTarget(network, address); err != nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
+	}
+	var err error
+	var c net.Conn
+	if d.ProxyDial != nil {
+		c, err = d.ProxyDial(context.Background(), d.proxyNetwork, d.proxyAddress)
+	} else {
+		c, err = net.Dial(d.proxyNetwork, d.proxyAddress)
+	}
+	if err != nil {
+		proxy, dst, _ := d.pathAddrs(address)
+		return nil, &net.OpError{Op: d.cmd.String(), Net: network, Source: proxy, Addr: dst, Err: err}
+	}
+	if _, err := d.DialWithConn(context.Background(), c, network, address); err != nil {
+		c.Close()
+		return nil, err
+	}
+	return c, nil
+}
+
+func (d *Dialer) validateTarget(network, address string) error {
+	switch network {
+	case "tcp", "tcp6", "tcp4":
+	default:
+		return errors.New("network not implemented")
+	}
+	switch d.cmd {
+	case CmdConnect, cmdBind:
+	default:
+		return errors.New("command not implemented")
+	}
+	return nil
+}
+
+func (d *Dialer) pathAddrs(address string) (proxy, dst net.Addr, err error) {
+	for i, s := range []string{d.proxyAddress, address} {
+		host, port, err := splitHostPort(s)
+		if err != nil {
+			return nil, nil, err
+		}
+		a := &Addr{Port: port}
+		a.IP = net.ParseIP(host)
+		if a.IP == nil {
+			a.Name = host
+		}
+		if i == 0 {
+			proxy = a
+		} else {
+			dst = a
+		}
+	}
+	return
+}
+
+// NewDialer returns a new Dialer that dials through the provided
+// proxy server's network and address.
+func NewDialer(network, address string) *Dialer {
+	return &Dialer{proxyNetwork: network, proxyAddress: address, cmd: CmdConnect}
+}
+
+const (
+	authUsernamePasswordVersion = 0x01
+	authStatusSucceeded         = 0x00
+)
+
+// UsernamePassword are the credentials for the username/password
+// authentication method.
+type UsernamePassword struct {
+	Username string
+	Password string
+}
+
+// Authenticate authenticates a pair of username and password with the
+// proxy server.
+func (up *UsernamePassword) Authenticate(ctx context.Context, rw io.ReadWriter, auth AuthMethod) error {
+	switch auth {
+	case AuthMethodNotRequired:
+		return nil
+	case AuthMethodUsernamePassword:
+		if len(up.Username) == 0 || len(up.Username) > 255 || len(up.Password) == 0 || len(up.Password) > 255 {
+			return errors.New("invalid username/password")
+		}
+		b := []byte{authUsernamePasswordVersion}
+		b = append(b, byte(len(up.Username)))
+		b = append(b, up.Username...)
+		b = append(b, byte(len(up.Password)))
+		b = append(b, up.Password...)
+		// TODO(mikio): handle IO deadlines and cancelation if
+		// necessary
+		if _, err := rw.Write(b); err != nil {
+			return err
+		}
+		if _, err := io.ReadFull(rw, b[:2]); err != nil {
+			return err
+		}
+		if b[0] != authUsernamePasswordVersion {
+			return errors.New("invalid username/password version")
+		}
+		if b[1] != authStatusSucceeded {
+			return errors.New("username/password authentication failed")
+		}
+		return nil
+	}
+	return errors.New("unsupported authentication method " + strconv.Itoa(int(auth)))
+}

vendor/golang.org/x/net/proxy/direct.go

@@ -0,0 +1,18 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+	"net"
+)
+
+type direct struct{}
+
+// Direct is a direct proxy: one that makes network connections directly.
+var Direct = direct{}
+
+func (direct) Dial(network, addr string) (net.Conn, error) {
+	return net.Dial(network, addr)
+}

vendor/golang.org/x/net/proxy/per_host.go

@@ -0,0 +1,140 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+	"net"
+	"strings"
+)
+
+// A PerHost directs connections to a default Dialer unless the host name
+// requested matches one of a number of exceptions.
+type PerHost struct {
+	def, bypass Dialer
+
+	bypassNetworks []*net.IPNet
+	bypassIPs      []net.IP
+	bypassZones    []string
+	bypassHosts    []string
+}
+
+// NewPerHost returns a PerHost Dialer that directs connections to either
+// defaultDialer or bypass, depending on whether the connection matches one of
+// the configured rules.
+func NewPerHost(defaultDialer, bypass Dialer) *PerHost {
+	return &PerHost{
+		def:    defaultDialer,
+		bypass: bypass,
+	}
+}
+
+// Dial connects to the address addr on the given network through either
+// defaultDialer or bypass.
+func (p *PerHost) Dial(network, addr string) (c net.Conn, err error) {
+	host, _, err := net.SplitHostPort(addr)
+	if err != nil {
+		return nil, err
+	}
+
+	return p.dialerForRequest(host).Dial(network, addr)
+}
+
+func (p *PerHost) dialerForRequest(host string) Dialer {
+	if ip := net.ParseIP(host); ip != nil {
+		for _, net := range p.bypassNetworks {
+			if net.Contains(ip) {
+				return p.bypass
+			}
+		}
+		for _, bypassIP := range p.bypassIPs {
+			if bypassIP.Equal(ip) {
+				return p.bypass
+			}
+		}
+		return p.def
+	}
+
+	for _, zone := range p.bypassZones {
+		if strings.HasSuffix(host, zone) {
+			return p.bypass
+		}
+		if host == zone[1:] {
+			// For a zone ".example.com", we match "example.com"
+			// too.
+			return p.bypass
+		}
+	}
+	for _, bypassHost := range p.bypassHosts {
+		if bypassHost == host {
+			return p.bypass
+		}
+	}
+	return p.def
+}
+
+// AddFromString parses a string that contains comma-separated values
+// specifying hosts that should use the bypass proxy. Each value is either an
+// IP address, a CIDR range, a zone (*.example.com) or a host name
+// (localhost). A best effort is made to parse the string and errors are
+// ignored.
+func (p *PerHost) AddFromString(s string) {
+	hosts := strings.Split(s, ",")
+	for _, host := range hosts {
+		host = strings.TrimSpace(host)
+		if len(host) == 0 {
+			continue
+		}
+		if strings.Contains(host, "/") {
+			// We assume that it's a CIDR address like 127.0.0.0/8
+			if _, net, err := net.ParseCIDR(host); err == nil {
+				p.AddNetwork(net)
+			}
+			continue
+		}
+		if ip := net.ParseIP(host); ip != nil {
+			p.AddIP(ip)
+			continue
+		}
+		if strings.HasPrefix(host, "*.") {
+			p.AddZone(host[1:])
+			continue
+		}
+		p.AddHost(host)
+	}
+}
+
+// AddIP specifies an IP address that will use the bypass proxy. Note that
+// this will only take effect if a literal IP address is dialed. A connection
+// to a named host will never match an IP.
+func (p *PerHost) AddIP(ip net.IP) {
+	p.bypassIPs = append(p.bypassIPs, ip)
+}
+
+// AddNetwork specifies an IP range that will use the bypass proxy. Note that
+// this will only take effect if a literal IP address is dialed. A connection
+// to a named host will never match.
+func (p *PerHost) AddNetwork(net *net.IPNet) {
+	p.bypassNetworks = append(p.bypassNetworks, net)
+}
+
+// AddZone specifies a DNS suffix that will use the bypass proxy. A zone of
+// "example.com" matches "example.com" and all of its subdomains.
+func (p *PerHost) AddZone(zone string) {
+	if strings.HasSuffix(zone, ".") {
+		zone = zone[:len(zone)-1]
+	}
+	if !strings.HasPrefix(zone, ".") {
+		zone = "." + zone
+	}
+	p.bypassZones = append(p.bypassZones, zone)
+}
+
+// AddHost specifies a host name that will use the bypass proxy.
+func (p *PerHost) AddHost(host string) {
+	if strings.HasSuffix(host, ".") {
+		host = host[:len(host)-1]
+	}
+	p.bypassHosts = append(p.bypassHosts, host)
+}

vendor/golang.org/x/net/proxy/proxy.go

@@ -0,0 +1,139 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package proxy provides support for a variety of protocols to proxy network
+// data.
+package proxy // import "golang.org/x/net/proxy"
+
+import (
+	"errors"
+	"net"
+	"net/url"
+	"os"
+	"sync"
+)
+
+// A Dialer is a means to establish a connection.
+type Dialer interface {
+	// Dial connects to the given address via the proxy.
+	Dial(network, addr string) (c net.Conn, err error)
+}
+
+// Auth contains authentication parameters that specific Dialers may require.
+type Auth struct {
+	User, Password string
+}
+
+// FromEnvironment returns the dialer specified by the proxy related variables in
+// the environment.
+func FromEnvironment() Dialer {
+	allProxy := allProxyEnv.Get()
+	if len(allProxy) == 0 {
+		return Direct
+	}
+
+	proxyURL, err := url.Parse(allProxy)
+	if err != nil {
+		return Direct
+	}
+	proxy, err := FromURL(proxyURL, Direct)
+	if err != nil {
+		return Direct
+	}
+
+	noProxy := noProxyEnv.Get()
+	if len(noProxy) == 0 {
+		return proxy
+	}
+
+	perHost := NewPerHost(proxy, Direct)
+	perHost.AddFromString(noProxy)
+	return perHost
+}
+
+// proxySchemes is a map from URL schemes to a function that creates a Dialer
+// from a URL with such a scheme.
+var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)
+
+// RegisterDialerType takes a URL scheme and a function to generate Dialers from
+// a URL with that scheme and a forwarding Dialer. Registered schemes are used
+// by FromURL.
+func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
+	if proxySchemes == nil {
+		proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
+	}
+	proxySchemes[scheme] = f
+}
+
+// FromURL returns a Dialer given a URL specification and an underlying
+// Dialer for it to make network requests.
+func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
+	var auth *Auth
+	if u.User != nil {
+		auth = new(Auth)
+		auth.User = u.User.Username()
+		if p, ok := u.User.Password(); ok {
+			auth.Password = p
+		}
+	}
+
+	switch u.Scheme {
+	case "socks5", "socks5h":
+		addr := u.Hostname()
+		port := u.Port()
+		if port == "" {
+			port = "1080"
+		}
+		return SOCKS5("tcp", net.JoinHostPort(addr, port), auth, forward)
+	}
+
+	// If the scheme doesn't match any of the built-in schemes, see if it
+	// was registered by another package.
+	if proxySchemes != nil {
+		if f, ok := proxySchemes[u.Scheme]; ok {
+			return f(u, forward)
+		}
+	}
+
+	return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
+}
+
+var (
+	allProxyEnv = &envOnce{
+		names: []string{"ALL_PROXY", "all_proxy"},
+	}
+	noProxyEnv = &envOnce{
+		names: []string{"NO_PROXY", "no_proxy"},
+	}
+)
+
+// envOnce looks up an environment variable (optionally by multiple
+// names) once. It mitigates expensive lookups on some platforms
+// (e.g. Windows).
+// (Borrowed from net/http/transport.go)
+type envOnce struct {
+	names []string
+	once  sync.Once
+	val   string
+}
+
+func (e *envOnce) Get() string {
+	e.once.Do(e.init)
+	return e.val
+}
+
+func (e *envOnce) init() {
+	for _, n := range e.names {
+		e.val = os.Getenv(n)
+		if e.val != "" {
+			return
+		}
+	}
+}
+
+// reset is used by tests
+func (e *envOnce) reset() {
+	e.once = sync.Once{}
+	e.val = ""
+}

vendor/golang.org/x/net/proxy/socks5.go

@@ -0,0 +1,36 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+	"context"
+	"net"
+
+	"golang.org/x/net/internal/socks"
+)
+
+// SOCKS5 returns a Dialer that makes SOCKSv5 connections to the given
+// address with an optional username and password.
+// See RFC 1928 and RFC 1929.
+func SOCKS5(network, address string, auth *Auth, forward Dialer) (Dialer, error) {
+	d := socks.NewDialer(network, address)
+	if forward != nil {
+		d.ProxyDial = func(_ context.Context, network string, address string) (net.Conn, error) {
+			return forward.Dial(network, address)
+		}
+	}
+	if auth != nil {
+		up := socks.UsernamePassword{
+			Username: auth.User,
+			Password: auth.Password,
+		}
+		d.AuthMethods = []socks.AuthMethod{
+			socks.AuthMethodNotRequired,
+			socks.AuthMethodUsernamePassword,
+		}
+		d.Authenticate = up.Authenticate
+	}
+	return d, nil
+}

vendor/vendor.json

@@ -257,10 +257,10 @@
 			"revisionTime": "2016-08-05T00:47:13Z"
 		},
 		{
-			"checksumSHA1": "LHwm1G0lFyQ3X0iHZ/MOYnAtjd4=",
+			"checksumSHA1": "VPQEUnKynzSvgugyvL3rxjduidg=",
 			"path": "github.com/eclipse/paho.mqtt.golang",
-			"revision": "d06cc70ac43d625e602946b5ff2346ddebb768e4",
-			"revisionTime": "2017-06-02T16:30:32Z"
+			"revision": "379fd9f99ba5b1f02c9fffb5e5952416ef9301dc",
+			"revisionTime": "2018-11-29T14:54:54Z"
 		},
 		{
 			"checksumSHA1": "g2AaO9VMaxzFQZkrCfxBUV3yAyM=",
@@ -698,10 +698,10 @@
 			"revisionTime": "2018-11-15T04:56:45Z"
 		},
 		{
-			"checksumSHA1": "LQSZ9jpJwWrfqKGiBnw+qjRtwRE=",
+			"checksumSHA1": "pxgHNx36gpRdhSqtaE5fqp7lrAA=",
 			"path": "github.com/minio/parquet-go",
-			"revision": "84e7daee0cea0bf82d1d15795dcea0f7bcd508c9",
-			"revisionTime": "2018-12-30T03:24:02Z"
+			"revision": "1014bfb4d0e323e3fbf6683e3519a98b0721f5cc",
+			"revisionTime": "2019-01-14T09:43:57Z"
 		},
 		{
 			"checksumSHA1": "N4WRPw4p3AN958RH/O53kUsJacQ=",
@@ -1200,6 +1200,12 @@
 			"revision": "da118f7b8e5954f39d0d2130ab35d4bf0e3cb344",
 			"revisionTime": "2017-04-23T14:02:46Z"
 		},
+		{
+			"checksumSHA1": "f3Y7JIZH61oMmp8nphqe8Mg+XoU=",
+			"path": "golang.org/x/net/internal/socks",
+			"revision": "1e06a53dbb7e2ed46e91183f219db23c6943c532",
+			"revisionTime": "2018-12-20T03:20:21Z"
+		},
 		{
 			"checksumSHA1": "UxahDzW2v4mf/+aFxruuupaoIwo=",
 			"path": "golang.org/x/net/internal/timeseries",
@@ -1212,6 +1218,12 @@
 			"revision": "da118f7b8e5954f39d0d2130ab35d4bf0e3cb344",
 			"revisionTime": "2017-04-23T14:02:46Z"
 		},
+		{
+			"checksumSHA1": "mCMW3hvbWFW1k5il9yyO7ELOdws=",
+			"path": "golang.org/x/net/proxy",
+			"revision": "1e06a53dbb7e2ed46e91183f219db23c6943c532",
+			"revisionTime": "2018-12-20T03:20:21Z"
+		},
 		{
 			"checksumSHA1": "9EZG3s2eOREO7WkBvigjk57wK/8=",
 			"path": "golang.org/x/net/trace",