Add admin API to send trace notifications to registered (#7128)

Remove current functionality to log trace to file
using MINIO_HTTP_TRACE env, and replace it with
mc admin trace command on mc client.

pkg/madmin/README.md

@@ -45,7 +45,7 @@ func main() {
 |:------------------------------------------|:--------------------------------------------|:-------------------|:----------------------------------|:------------------------|:--------------------------------------|:--------------------------------------------------|
 | [`ServiceStatus`](#ServiceStatus)         | [`ServerInfo`](#ServerInfo)                 | [`Heal`](#Heal)    | [`GetConfig`](#GetConfig)         | [`TopLocks`](#TopLocks) | [`AddUser`](#AddUser)                 |                                                   |
 | [`ServiceSendAction`](#ServiceSendAction) | [`ServerCPULoadInfo`](#ServerCPULoadInfo)   |                    | [`SetConfig`](#SetConfig)         |                         | [`SetUserPolicy`](#SetUserPolicy)     | [`StartProfiling`](#StartProfiling)               |
-|                                           | [`ServerMemUsageInfo`](#ServerMemUsageInfo) |                    | [`GetConfigKeys`](#GetConfigKeys) |                         | [`ListUsers`](#ListUsers)             | [`DownloadProfilingData`](#DownloadProfilingData) |
+| [`Trace`](#Trace)                                          | [`ServerMemUsageInfo`](#ServerMemUsageInfo) |                    | [`GetConfigKeys`](#GetConfigKeys) |                         | [`ListUsers`](#ListUsers)             | [`DownloadProfilingData`](#DownloadProfilingData) |
 |                                           |                                             |                    | [`SetConfigKeys`](#SetConfigKeys) |                         | [`AddCannedPolicy`](#AddCannedPolicy) |                                                   |
 
 
@@ -537,3 +537,22 @@ __Example__
 
     log.Println("Profiling data successfully downloaded.")
 ```
+
+<a name="Trace"></a>
+### Trace(allTrace bool,doneCh <-chan struct{}) <-chan TraceInfo
+Enable HTTP request tracing on all nodes in a MinIO cluster
+
+__Example__
+
+``` go
+    doneCh := make(chan struct{})
+    defer close(doneCh)
+    // listen to all trace including internal API calls
+    allTrace := true
+    // Start listening on all trace activity.
+    traceCh := madmClnt.Trace(allTrace,doneCh)
+    for traceInfo := range traceCh {
+        fmt.Println(traceInfo.String())
+    }
+    log.Println("Success")
+```

pkg/madmin/api-trace.go

@@ -0,0 +1,92 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package madmin
+
+import (
+	"bufio"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	trace "github.com/minio/minio/pkg/trace"
+)
+
+// TraceInfo holds http trace
+type TraceInfo struct {
+	Trace trace.Info
+	Err   error `json:"-"`
+}
+
+// Trace - listen on http trace notifications.
+func (adm AdminClient) Trace(allTrace bool, doneCh <-chan struct{}) <-chan TraceInfo {
+	traceInfoCh := make(chan TraceInfo, 1)
+	// Only success, start a routine to start reading line by line.
+	go func(traceInfoCh chan<- TraceInfo) {
+		defer close(traceInfoCh)
+		for {
+			urlValues := make(url.Values)
+			urlValues.Set("all", strconv.FormatBool(allTrace))
+			reqData := requestData{
+				relPath:     "/v1/trace",
+				queryValues: urlValues,
+			}
+			// Execute GET to call trace handler
+			resp, err := adm.executeMethod("GET", reqData)
+			if err != nil {
+				closeResponse(resp)
+				return
+			}
+
+			if resp.StatusCode != http.StatusOK {
+				traceInfoCh <- TraceInfo{Err: httpRespToErrorResponse(resp)}
+				return
+			}
+
+			// Initialize a new bufio scanner, to read line by line.
+			bio := bufio.NewScanner(resp.Body)
+
+			// Close the response body.
+			defer resp.Body.Close()
+
+			// Unmarshal each line, returns marshaled values.
+			for bio.Scan() {
+				var traceRec trace.Info
+				if err = json.Unmarshal(bio.Bytes(), &traceRec); err != nil {
+					continue
+				}
+				select {
+				case <-doneCh:
+					return
+				case traceInfoCh <- TraceInfo{Trace: traceRec}:
+				}
+			}
+			// Look for any underlying errors.
+			if err = bio.Err(); err != nil {
+				// For an unexpected connection drop from server, we close the body
+				// and re-connect.
+				if err == io.ErrUnexpectedEOF {
+					resp.Body.Close()
+				}
+			}
+		}
+	}(traceInfoCh)
+
+	// Returns the trace info channel, for caller to start reading from.
+	return traceInfoCh
+}

pkg/madmin/examples/trace.go

@@ -0,0 +1,50 @@
+// +build ignore
+
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an MinIO Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	// Start listening on all http trace activity from all servers
+	// in the minio cluster.
+	traceCh := madmClnt.Trace(false, doneCh)
+	for traceInfo := range traceCh {
+		if traceInfo.Err != nil {
+			fmt.Println(traceInfo.Err)
+		}
+		fmt.Println(traceInfo)
+	}
+}