Migrate config to KV data format (#8392)

- adding oauth support to MinIO browser (#8400) by @kanagaraj
- supports multi-line get/set/del for all config fields
- add support for comments, allow toggle
- add extensive validation of config before saving
- support MinIO browser to support proper claims, using STS tokens
- env support for all config parameters, legacy envs are also
  supported with all documentation now pointing to latest ENVs
- preserve accessKey/secretKey from FS mode setups
- add history support implements three APIs
  - ClearHistory
  - RestoreHistory
  - ListHistory
- add help command support for each config parameters
- all the bug fixes after migration to KV, and other bug
  fixes encountered during testing.

pkg/madmin/README.md

@@ -617,8 +617,8 @@ __Example__
 ### GetKeyStatus(keyID string) (*KMSKeyStatus, error)
 Requests status information about one particular KMS master key
 from a MinIO server. The keyID is optional and the server will
-use the default master key (configured via `MINIO_SSE_VAULT_KEY_NAME`
-or `MINIO_SSE_MASTER_KEY`) if the keyID is empty.
+use the default master key (configured via `MINIO_KMS_VAULT_KEY_NAME`
+or `MINIO_KMS_MASTER_KEY`) if the keyID is empty.
 
 __Example__
 

pkg/madmin/config-commands.go

@@ -1,5 +1,5 @@
 /*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
+ * MinIO Cloud Storage, (C) 2017-2019 MinIO, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -19,18 +19,14 @@ package madmin
 
 import (
 	"bytes"
-	"encoding/json"
-	"errors"
 	"io"
 	"net/http"
-
-	"github.com/minio/minio/pkg/quick"
 )
 
 // GetConfig - returns the config.json of a minio setup, incoming data is encrypted.
 func (adm *AdminClient) GetConfig() ([]byte, error) {
 	// Execute GET on /minio/admin/v2/config to get config of a setup.
-	resp, err := adm.executeMethod("GET",
+	resp, err := adm.executeMethod(http.MethodGet,
 		requestData{relPath: adminAPIPrefix + "/config"})
 	defer closeResponse(resp)
 	if err != nil {
@@ -40,7 +36,6 @@ func (adm *AdminClient) GetConfig() ([]byte, error) {
 	if resp.StatusCode != http.StatusOK {
 		return nil, httpRespToErrorResponse(resp)
 	}
-	defer closeResponse(resp)
 
 	return DecryptData(adm.secretAccessKey, resp.Body)
 }
@@ -59,26 +54,6 @@ func (adm *AdminClient) SetConfig(config io.Reader) (err error) {
 		return err
 	}
 	configBytes := configBuf[:n]
-
-	type configVersion struct {
-		Version string `json:"version,omitempty"`
-	}
-	var cfg configVersion
-
-	// Check if read data is in json format
-	if err = json.Unmarshal(configBytes, &cfg); err != nil {
-		return errors.New("Invalid JSON format: " + err.Error())
-	}
-
-	// Check if the provided json file has "version" key set
-	if cfg.Version == "" {
-		return errors.New("Missing or unset \"version\" key in json file")
-	}
-	// Validate there are no duplicate keys in the JSON
-	if err = quick.CheckDuplicateKeys(string(configBytes)); err != nil {
-		return errors.New("Duplicate key in json file: " + err.Error())
-	}
-
 	econfigBytes, err := EncryptData(adm.secretAccessKey, configBytes)
 	if err != nil {
 		return err
@@ -90,7 +65,7 @@ func (adm *AdminClient) SetConfig(config io.Reader) (err error) {
 	}
 
 	// Execute PUT on /minio/admin/v2/config to set config.
-	resp, err := adm.executeMethod("PUT", reqData)
+	resp, err := adm.executeMethod(http.MethodPut, reqData)
 
 	defer closeResponse(resp)
 	if err != nil {

pkg/madmin/config-help-commands.go

@@ -0,0 +1,49 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"io"
+	"net/http"
+	"net/url"
+)
+
+// HelpConfigKV - return help for a given sub-system.
+func (adm *AdminClient) HelpConfigKV(subSys, key string) (io.ReadCloser, error) {
+	v := url.Values{}
+	v.Set("subSys", subSys)
+	v.Set("key", key)
+	reqData := requestData{
+		relPath:     adminAPIPrefix + "/help-config-kv",
+		queryValues: v,
+	}
+
+	// Execute GET on /minio/admin/v2/help-config-kv
+	resp, err := adm.executeMethod(http.MethodGet, reqData)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		defer closeResponse(resp)
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	return resp.Body, nil
+
+}

pkg/madmin/config-history-commands.go

@@ -0,0 +1,107 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/url"
+	"time"
+)
+
+// ClearConfigHistoryKV - clears the config entry represented by restoreID.
+// optionally allows setting `all` as a special keyword to automatically
+// erase all config set history entires.
+func (adm *AdminClient) ClearConfigHistoryKV(restoreID string) (err error) {
+	v := url.Values{}
+	v.Set("restoreId", restoreID)
+	reqData := requestData{
+		relPath:     adminAPIPrefix + "/clear-config-history-kv",
+		queryValues: v,
+	}
+
+	// Execute DELETE on /minio/admin/v2/clear-config-history-kv
+	resp, err := adm.executeMethod(http.MethodDelete, reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return httpRespToErrorResponse(resp)
+	}
+
+	return nil
+}
+
+// RestoreConfigHistoryKV - Restore a previous config set history.
+// Input is a unique id which represents the previous setting.
+func (adm *AdminClient) RestoreConfigHistoryKV(restoreID string) (err error) {
+	v := url.Values{}
+	v.Set("restoreId", restoreID)
+	reqData := requestData{
+		relPath:     adminAPIPrefix + "/restore-config-history-kv",
+		queryValues: v,
+	}
+
+	// Execute PUT on /minio/admin/v2/set-config-kv to set config key/value.
+	resp, err := adm.executeMethod(http.MethodPut, reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return httpRespToErrorResponse(resp)
+	}
+
+	return nil
+}
+
+// ConfigHistoryEntry - captures config set history with a unique
+// restore ID and createTime
+type ConfigHistoryEntry struct {
+	RestoreID  string    `json:"restoreId"`
+	CreateTime time.Time `json:"createTime"`
+}
+
+// ListConfigHistoryKV - lists a slice of ConfigHistoryEntries sorted by createTime.
+func (adm *AdminClient) ListConfigHistoryKV() ([]ConfigHistoryEntry, error) {
+	// Execute GET on /minio/admin/v2/list-config-history-kv
+	resp, err := adm.executeMethod(http.MethodGet,
+		requestData{
+			relPath: adminAPIPrefix + "/list-config-history-kv",
+		})
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	var chEntries []ConfigHistoryEntry
+	d := json.NewDecoder(resp.Body)
+	d.DisallowUnknownFields()
+	if err = d.Decode(&chEntries); err != nil {
+		return chEntries, err
+	}
+	return chEntries, nil
+}

pkg/madmin/config-kv-commands.go

@@ -0,0 +1,100 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"net/http"
+	"net/url"
+)
+
+// DelConfigKV - delete key from server config.
+func (adm *AdminClient) DelConfigKV(k string) (err error) {
+	econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(k))
+	if err != nil {
+		return err
+	}
+
+	reqData := requestData{
+		relPath: adminAPIPrefix + "/del-config-kv",
+		content: econfigBytes,
+	}
+
+	// Execute DELETE on /minio/admin/v2/del-config-kv to delete config key.
+	resp, err := adm.executeMethod(http.MethodDelete, reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return httpRespToErrorResponse(resp)
+	}
+
+	return nil
+}
+
+// SetConfigKV - set key value config to server.
+func (adm *AdminClient) SetConfigKV(kv string) (err error) {
+	econfigBytes, err := EncryptData(adm.secretAccessKey, []byte(kv))
+	if err != nil {
+		return err
+	}
+
+	reqData := requestData{
+		relPath: adminAPIPrefix + "/set-config-kv",
+		content: econfigBytes,
+	}
+
+	// Execute PUT on /minio/admin/v2/set-config-kv to set config key/value.
+	resp, err := adm.executeMethod(http.MethodPut, reqData)
+
+	defer closeResponse(resp)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return httpRespToErrorResponse(resp)
+	}
+
+	return nil
+}
+
+// GetConfigKV - returns the key, value of the requested key, incoming data is encrypted.
+func (adm *AdminClient) GetConfigKV(key string) ([]byte, error) {
+	v := url.Values{}
+	v.Set("key", key)
+
+	// Execute GET on /minio/admin/v2/get-config-kv?key={key} to get value of key.
+	resp, err := adm.executeMethod(http.MethodGet,
+		requestData{
+			relPath:     adminAPIPrefix + "/get-config-kv",
+			queryValues: v,
+		})
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	return DecryptData(adm.secretAccessKey, resp.Body)
+}

pkg/madmin/examples/get-config.go

@@ -1,54 +0,0 @@
-// +build ignore
-
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"log"
-
-	"github.com/minio/minio/pkg/madmin"
-)
-
-func main() {
-	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
-	// dummy values, please replace them with original values.
-
-	// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
-	// New returns an MinIO Admin client object.
-	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	configBytes, err := madmClnt.GetConfig()
-	if err != nil {
-		log.Fatalf("failed due to: %v", err)
-	}
-
-	// Pretty-print config received as json.
-	var buf bytes.Buffer
-	err = json.Indent(&buf, configBytes, "", "\t")
-	if err != nil {
-		log.Fatalf("failed due to: %v", err)
-	}
-
-	log.Println("config received successfully: ", string(buf.Bytes()))
-}

pkg/madmin/examples/set-config.go

@@ -1,155 +0,0 @@
-// +build ignore
-
-/*
- * MinIO Cloud Storage, (C) 2017 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-package main
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"log"
-
-	"github.com/minio/minio/pkg/madmin"
-)
-
-var configJSON = []byte(`{
-	"version": "13",
-	"credential": {
-		"accessKey": "minio",
-		"secretKey": "minio123"
-	},
-	"region": "us-west-1",
-	"logger": {
-		"console": {
-			"enable": true,
-			"level": "fatal"
-		},
-		"file": {
-			"enable": false,
-			"fileName": "",
-			"level": ""
-		}
-	},
-	"notify": {
-		"amqp": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"exchange": "",
-				"routingKey": "",
-				"exchangeType": "",
-				"mandatory": false,
-				"immediate": false,
-				"durable": false,
-				"internal": false,
-				"noWait": false,
-				"autoDeleted": false
-			}
-		},
-		"nats": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"subject": "",
-				"username": "",
-				"password": "",
-				"token": "",
-				"secure": false,
-				"pingInterval": 0,
-				"streaming": {
-					"enable": false,
-					"clusterID": "",
-					"async": false,
-					"maxPubAcksInflight": 0
-				}
-			}
-		},
-		"elasticsearch": {
-			"1": {
-				"enable": false,
-				"url": "",
-				"index": ""
-			}
-		},
-		"redis": {
-			"1": {
-				"enable": false,
-				"address": "",
-				"password": "",
-				"key": ""
-			}
-		},
-		"postgresql": {
-			"1": {
-				"enable": false,
-				"connectionString": "",
-				"table": "",
-				"host": "",
-				"port": "",
-				"user": "",
-				"password": "",
-				"database": ""
-			}
-		},
-		"kafka": {
-			"1": {
-				"enable": false,
-				"brokers": null,
-				"topic": ""
-			}
-		},
-		"webhook": {
-			"1": {
-				"enable": false,
-				"endpoint": ""
-			}
-		}
-	}
-}`)
-
-func main() {
-	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
-	// dummy values, please replace them with original values.
-
-	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
-	// dummy values, please replace them with original values.
-
-	// API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise.
-	// New returns an MinIO Admin client object.
-	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	result, err := madmClnt.SetConfig(bytes.NewReader(configJSON))
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	var buf bytes.Buffer
-	enc := json.NewEncoder(&buf)
-	enc.SetEscapeHTML(false)
-	enc.SetIndent("", "\t")
-	err = enc.Encode(result)
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	fmt.Println("SetConfig: ", string(buf.Bytes()))
-}