Add etcd part of config support, add noColor/json support (#8439)

- Add color/json mode support for get/help commands
- Support ENV help for all sub-systems
- Add support for etcd as part of config

pkg/madmin/config-help-commands.go

@@ -18,16 +18,50 @@
 package madmin
 
 import (
+	"encoding/json"
 	"io"
 	"net/http"
 	"net/url"
+	"strings"
+	"text/tabwriter"
+	"text/template"
+
+	"github.com/minio/minio/pkg/color"
 )
 
+// Help template used by all sub-systems
+const Help = `{{colorBlueBold "Key"}}{{"\t"}}{{colorBlueBold "Description"}}
+{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
+{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
+{{end}}`
+
+// HelpEnv template used by all sub-systems
+const HelpEnv = `{{colorBlueBold "KeyEnv"}}{{"\t"}}{{colorBlueBold "Description"}}
+{{colorYellowBold "----"}}{{"\t"}}{{colorYellowBold "----"}}
+{{range $key, $value := .}}{{colorCyanBold $key}}{{ "\t" }}{{$value}}
+{{end}}`
+
+var funcMap = template.FuncMap{
+	"colorBlueBold":   color.BlueBold,
+	"colorYellowBold": color.YellowBold,
+	"colorCyanBold":   color.CyanBold,
+}
+
+// HelpTemplate - captures config help template
+var HelpTemplate = template.Must(template.New("config-help").Funcs(funcMap).Parse(Help))
+
+// HelpEnvTemplate - captures config help template
+var HelpEnvTemplate = template.Must(template.New("config-help-env").Funcs(funcMap).Parse(HelpEnv))
+
 // HelpConfigKV - return help for a given sub-system.
-func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error) {
+func (adm *AdminClient) HelpConfigKV(subSys, key string, envOnly bool) (io.Reader, error) {
 	v := url.Values{}
 	v.Set("subSys", subSys)
 	v.Set("key", key)
+	if envOnly {
+		v.Set("env", "")
+	}
+
 	reqData := requestData{
 		relPath:     adminAPIPrefix + "/help-config-kv",
 		queryValues: v,
@@ -38,12 +72,29 @@ func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error)
 	if err != nil {
 		return nil, err
 	}
+	defer closeResponse(resp)
 
 	if resp.StatusCode != http.StatusOK {
-		defer closeResponse(resp)
 		return nil, httpRespToErrorResponse(resp)
 	}
 
-	return resp.Body, nil
+	var help = make(map[string]string)
+	d := json.NewDecoder(resp.Body)
+	if err = d.Decode(&help); err != nil {
+		return nil, err
+	}
 
+	var s strings.Builder
+	w := tabwriter.NewWriter(&s, 1, 8, 2, ' ', 0)
+	if !envOnly {
+		err = HelpTemplate.Execute(w, help)
+	} else {
+		err = HelpEnvTemplate.Execute(w, help)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	w.Flush()
+	return strings.NewReader(s.String()), nil
 }

pkg/madmin/config-kv-commands.go

@@ -18,8 +18,11 @@
 package madmin
 
 import (
+	"bufio"
+	"encoding/base64"
 	"net/http"
 	"net/url"
+	"strings"
 )
 
 // DelConfigKV - delete key from server config.
@@ -51,7 +54,32 @@ func (adm *AdminClient) DelConfigKV(k string) (err error) {
 
 // SetConfigKV - set key value config to server.
 func (adm *AdminClient) SetConfigKV(kv string) (err error) {
-	econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(kv))
+	bio := bufio.NewScanner(strings.NewReader(kv))
+	var s strings.Builder
+	var comment string
+	for bio.Scan() {
+		if bio.Text() == "" {
+			continue
+		}
+		if strings.HasPrefix(bio.Text(), KvComment) {
+			// Join multiple comments for each newline, separated by "\n"
+			comments := []string{comment, strings.TrimPrefix(bio.Text(), KvComment)}
+			comment = strings.Join(comments, KvNewline)
+			continue
+		}
+		s.WriteString(bio.Text())
+		if comment != "" {
+			s.WriteString(KvSpaceSeparator)
+			s.WriteString(commentKey)
+			s.WriteString(KvSeparator)
+			s.WriteString(KvDoubleQuote)
+			s.WriteString(base64.RawStdEncoding.EncodeToString([]byte(comment)))
+			s.WriteString(KvDoubleQuote)
+		}
+		comment = ""
+	}
+
+	econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(s.String()))
 	if err != nil {
 		return err
 	}
@@ -77,7 +105,7 @@ func (adm *AdminClient) SetConfigKV(kv string) (err error) {
 }
 
 // GetConfigKV - returns the key, value of the requested key, incoming data is encrypted.
-func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
+func (adm *AdminClient) GetConfigKV(key string) (Targets, error) {
 	v := url.Values{}
 	v.Set("key", key)
 
@@ -92,9 +120,16 @@ func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
 		return nil, err
 	}
 
+	defer closeResponse(resp)
+
 	if resp.StatusCode != http.StatusOK {
 		return nil, httpRespToErrorResponse(resp)
 	}
 
-	return DecryptData(adm.secretAccessKey, resp.Body)
+	data, err := DecryptData(adm.secretAccessKey, resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return parseSubSysTarget(data)
 }

pkg/madmin/parse-kv.go

@@ -0,0 +1,156 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/pkg/color"
+)
+
+// KVS each sub-system key, value
+type KVS map[string]string
+
+// Targets sub-system targets
+type Targets map[string]map[string]KVS
+
+const (
+	commentKey = "comment"
+)
+
+func (t Targets) String() string {
+	var s strings.Builder
+	for subSys, targetKV := range t {
+		for target, kv := range targetKV {
+			c := kv[commentKey]
+			data, err := base64.RawStdEncoding.DecodeString(c)
+			if err == nil {
+				c = string(data)
+			}
+			for _, c1 := range strings.Split(c, KvNewline) {
+				if c1 == "" {
+					continue
+				}
+				s.WriteString(color.YellowBold(KvComment))
+				s.WriteString(KvSpaceSeparator)
+				s.WriteString(color.BlueBold(strings.TrimSpace(c1)))
+				s.WriteString(KvNewline)
+			}
+			s.WriteString(subSys)
+			if target != Default {
+				s.WriteString(SubSystemSeparator)
+				s.WriteString(target)
+			}
+			s.WriteString(KvSpaceSeparator)
+			for k, v := range kv {
+				// Comment is already printed, do not print it here.
+				if k == commentKey {
+					continue
+				}
+				s.WriteString(k)
+				s.WriteString(KvSeparator)
+				s.WriteString(KvDoubleQuote)
+				s.WriteString(v)
+				s.WriteString(KvDoubleQuote)
+				s.WriteString(KvSpaceSeparator)
+			}
+			if len(t) > 1 {
+				s.WriteString(KvNewline)
+				s.WriteString(KvNewline)
+			}
+		}
+	}
+	return s.String()
+}
+
+// Constant separators
+const (
+	SubSystemSeparator = `:`
+	KvSeparator        = `=`
+	KvSpaceSeparator   = ` `
+	KvComment          = `#`
+	KvDoubleQuote      = `"`
+	KvSingleQuote      = `'`
+
+	KvNewline = "\n"
+	Default   = `_`
+)
+
+// This function is needed, to trim off single or double quotes, creeping into the values.
+func sanitizeValue(v string) string {
+	v = strings.TrimSuffix(strings.TrimPrefix(strings.TrimSpace(v), KvDoubleQuote), KvDoubleQuote)
+	return strings.TrimSuffix(strings.TrimPrefix(v, KvSingleQuote), KvSingleQuote)
+}
+
+func convertTargets(s string, targets Targets) error {
+	inputs := strings.SplitN(s, KvSpaceSeparator, 2)
+	if len(inputs) <= 1 {
+		return fmt.Errorf("invalid number of arguments '%s'", s)
+	}
+	subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2)
+	if len(subSystemValue) == 0 {
+		return fmt.Errorf("invalid number of arguments %s", s)
+	}
+	var kvs = KVS{}
+	var prevK string
+	for _, v := range strings.Fields(inputs[1]) {
+		kv := strings.SplitN(v, KvSeparator, 2)
+		if len(kv) == 0 {
+			continue
+		}
+		if len(kv) == 1 && prevK != "" {
+			kvs[prevK] = strings.Join([]string{kvs[prevK], sanitizeValue(kv[0])}, KvSpaceSeparator)
+			continue
+		}
+		if len(kv[1]) == 0 {
+			return fmt.Errorf("value for key '%s' cannot be empty", kv[0])
+		}
+		prevK = kv[0]
+		kvs[kv[0]] = sanitizeValue(kv[1])
+	}
+
+	_, ok := targets[subSystemValue[0]]
+	if !ok {
+		targets[subSystemValue[0]] = map[string]KVS{}
+	}
+	if len(subSystemValue) == 2 {
+		targets[subSystemValue[0]][subSystemValue[1]] = kvs
+	} else {
+		targets[subSystemValue[0]][Default] = kvs
+	}
+	return nil
+}
+
+// parseSubSysTarget - parse sub-system target
+func parseSubSysTarget(buf []byte) (Targets, error) {
+	targets := make(map[string]map[string]KVS)
+	bio := bufio.NewScanner(bytes.NewReader(buf))
+	for bio.Scan() {
+		if err := convertTargets(bio.Text(), targets); err != nil {
+			return nil, err
+		}
+	}
+	if err := bio.Err(); err != nil {
+		return nil, err
+	}
+	return targets, nil
+}