Refactor config and split them in packages (#8351)

This change is related to larger config migration PR
change, this is a first stage change to move our
configs to `cmd/config/` - divided into its subsystems

cmd/config/cache/config.go

@@ -0,0 +1,118 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"encoding/json"
+	"errors"
+	"path/filepath"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/ellipses"
+)
+
+// Config represents cache config settings
+type Config struct {
+	Drives  []string `json:"drives"`
+	Expiry  int      `json:"expiry"`
+	MaxUse  int      `json:"maxuse"`
+	Exclude []string `json:"exclude"`
+}
+
+// UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
+// json entries for CacheConfig.
+func (cfg *Config) UnmarshalJSON(data []byte) (err error) {
+	type Alias Config
+	var _cfg = &struct {
+		*Alias
+	}{
+		Alias: (*Alias)(cfg),
+	}
+	if err = json.Unmarshal(data, _cfg); err != nil {
+		return err
+	}
+
+	if _cfg.Expiry < 0 {
+		return errors.New("config expiry value should not be negative")
+	}
+
+	if _cfg.MaxUse < 0 {
+		return errors.New("config max use value should not be null or negative")
+	}
+
+	if _, err = parseCacheDrives(_cfg.Drives); err != nil {
+		return err
+	}
+	if _, err = parseCacheExcludes(_cfg.Exclude); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Parses given cacheDrivesEnv and returns a list of cache drives.
+func parseCacheDrives(drives []string) ([]string, error) {
+	if len(drives) == 0 {
+		return drives, nil
+	}
+	var endpoints []string
+	for _, d := range drives {
+		if ellipses.HasEllipses(d) {
+			s, err := parseCacheDrivePaths(d)
+			if err != nil {
+				return nil, err
+			}
+			endpoints = append(endpoints, s...)
+		} else {
+			endpoints = append(endpoints, d)
+		}
+	}
+
+	for _, d := range endpoints {
+		if !filepath.IsAbs(d) {
+			return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)
+		}
+	}
+	return endpoints, nil
+}
+
+// Parses all arguments and returns a slice of drive paths following the ellipses pattern.
+func parseCacheDrivePaths(arg string) (ep []string, err error) {
+	patterns, perr := ellipses.FindEllipsesPatterns(arg)
+	if perr != nil {
+		return []string{}, config.ErrInvalidCacheDrivesValue(nil).Msg(perr.Error())
+	}
+
+	for _, lbls := range patterns.Expand() {
+		ep = append(ep, strings.Join(lbls, ""))
+	}
+
+	return ep, nil
+}
+
+// Parses given cacheExcludesEnv and returns a list of cache exclude patterns.
+func parseCacheExcludes(excludes []string) ([]string, error) {
+	for _, e := range excludes {
+		if len(e) == 0 {
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude path (%s) cannot be empty", e)
+		}
+		if strings.HasPrefix(e, "/") {
+			return nil, config.ErrInvalidCacheExcludesValue(nil).Msg("cache exclude pattern (%s) cannot start with / as prefix", e)
+		}
+	}
+	return excludes, nil
+}

cmd/config/cache/config_test.go

@@ -0,0 +1,114 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"reflect"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+// Tests cache drive parsing.
+func TestParseCacheDrives(t *testing.T) {
+	testCases := []struct {
+		driveStr         string
+		expectedPatterns []string
+		success          bool
+	}{
+		// valid input
+
+		{"bucket1/*;*.png;images/trip/barcelona/*", []string{}, false},
+		{"bucket1", []string{}, false},
+	}
+	if runtime.GOOS == "windows" {
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive1;C:/home/drive2;C:/home/drive3", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1...3}", []string{"C:/home/drive1", "C:/home/drive2", "C:/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"C:/home/drive{1..3}", []string{}, false})
+	} else {
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive1;/home/drive2;/home/drive3", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1...3}", []string{"/home/drive1", "/home/drive2", "/home/drive3"}, true})
+		testCases = append(testCases, struct {
+			driveStr         string
+			expectedPatterns []string
+			success          bool
+		}{"/home/drive{1..3}", []string{}, false})
+	}
+	for i, testCase := range testCases {
+		drives, err := parseCacheDrives(strings.Split(testCase.driveStr, cacheEnvDelimiter))
+		if err != nil && testCase.success {
+			t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
+		}
+		if err == nil && !testCase.success {
+			t.Errorf("Test %d: Expected failure but passed instead", i+1)
+		}
+		if err == nil {
+			if !reflect.DeepEqual(drives, testCase.expectedPatterns) {
+				t.Errorf("Test %d: Expected %v, got %v", i+1, testCase.expectedPatterns, drives)
+			}
+		}
+	}
+}
+
+// Tests cache exclude parsing.
+func TestParseCacheExclude(t *testing.T) {
+	testCases := []struct {
+		excludeStr       string
+		expectedPatterns []string
+		success          bool
+	}{
+		// valid input
+		{"/home/drive1;/home/drive2;/home/drive3", []string{}, false},
+		{"bucket1/*;*.png;images/trip/barcelona/*", []string{"bucket1/*", "*.png", "images/trip/barcelona/*"}, true},
+		{"bucket1", []string{"bucket1"}, true},
+	}
+
+	for i, testCase := range testCases {
+		excludes, err := parseCacheExcludes(strings.Split(testCase.excludeStr, cacheEnvDelimiter))
+		if err != nil && testCase.success {
+			t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
+		}
+		if err == nil && !testCase.success {
+			t.Errorf("Test %d: Expected failure but passed instead", i+1)
+		}
+		if err == nil {
+			if !reflect.DeepEqual(excludes, testCase.expectedPatterns) {
+				t.Errorf("Test %d: Expected %v, got %v", i+1, testCase.expectedPatterns, excludes)
+			}
+		}
+	}
+}

cmd/config/cache/lookup.go

@@ -0,0 +1,79 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cache
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Cache ENVs
+const (
+	EnvCacheDrives              = "MINIO_CACHE_DRIVES"
+	EnvCacheExclude             = "MINIO_CACHE_EXCLUDE"
+	EnvCacheExpiry              = "MINIO_CACHE_EXPIRY"
+	EnvCacheMaxUse              = "MINIO_CACHE_MAXUSE"
+	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
+)
+
+const (
+	cacheEnvDelimiter = ";"
+)
+
+// LookupConfig - extracts cache configuration provided by environment
+// variables and merge them with provided CacheConfiguration.
+func LookupConfig(cfg Config) (Config, error) {
+	if drives := env.Get(EnvCacheDrives, strings.Join(cfg.Drives, ",")); drives != "" {
+		driveList, err := parseCacheDrives(strings.Split(drives, cacheEnvDelimiter))
+		if err != nil {
+			return cfg, err
+		}
+		cfg.Drives = driveList
+	}
+
+	if excludes := env.Get(EnvCacheExclude, strings.Join(cfg.Exclude, ",")); excludes != "" {
+		excludeList, err := parseCacheExcludes(strings.Split(excludes, cacheEnvDelimiter))
+		if err != nil {
+			return cfg, err
+		}
+		cfg.Exclude = excludeList
+	}
+
+	if expiryStr := env.Get(EnvCacheExpiry, strconv.Itoa(cfg.Expiry)); expiryStr != "" {
+		expiry, err := strconv.Atoi(expiryStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheExpiryValue(err)
+		}
+		cfg.Expiry = expiry
+	}
+
+	if maxUseStr := env.Get(EnvCacheMaxUse, strconv.Itoa(cfg.MaxUse)); maxUseStr != "" {
+		maxUse, err := strconv.Atoi(maxUseStr)
+		if err != nil {
+			return cfg, config.ErrInvalidCacheMaxUse(err)
+		}
+		// maxUse should be a valid percentage.
+		if maxUse > 0 && maxUse <= 100 {
+			cfg.MaxUse = maxUse
+		}
+	}
+
+	return cfg, nil
+}

cmd/config/compress/compress.go

@@ -0,0 +1,79 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package compress
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/env"
+)
+
+// Config represents the compression settings.
+type Config struct {
+	Enabled    bool     `json:"enabled"`
+	Extensions []string `json:"extensions"`
+	MimeTypes  []string `json:"mime-types"`
+}
+
+// Compression environment variables
+const (
+	EnvMinioCompress           = "MINIO_COMPRESS"
+	EnvMinioCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
+	EnvMinioCompressMimeTypes  = "MINIO_COMPRESS_MIMETYPES"
+)
+
+// Parses the given compression exclude list `extensions` or `content-types`.
+func parseCompressIncludes(includes []string) ([]string, error) {
+	for _, e := range includes {
+		if len(e) == 0 {
+			return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type (%s) cannot be empty", e)
+		}
+	}
+	return includes, nil
+}
+
+// LookupConfig - lookup compression config.
+func LookupConfig(cfg Config) (Config, error) {
+	const compressEnvDelimiter = ","
+	if compress := env.Get(EnvMinioCompress, strconv.FormatBool(cfg.Enabled)); compress != "" {
+		cfg.Enabled = strings.EqualFold(compress, "true")
+	}
+
+	compressExtensions := env.Get(EnvMinioCompressExtensions, strings.Join(cfg.Extensions, ","))
+	compressMimeTypes := env.Get(EnvMinioCompressMimeTypes, strings.Join(cfg.MimeTypes, ","))
+	if compressExtensions != "" || compressMimeTypes != "" {
+		if compressExtensions != "" {
+			extensions, err := parseCompressIncludes(strings.Split(compressExtensions, compressEnvDelimiter))
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions)
+			}
+			cfg.Extensions = extensions
+		}
+		if compressMimeTypes != "" {
+			contenttypes, err := parseCompressIncludes(strings.Split(compressMimeTypes, compressEnvDelimiter))
+			if err != nil {
+				return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, contenttypes)
+			}
+			cfg.MimeTypes = contenttypes
+		}
+	}
+
+	return cfg, nil
+}

cmd/config/errors-utls.go

@@ -0,0 +1,132 @@
+/*
+ * MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"syscall"
+
+	"github.com/minio/minio/pkg/color"
+)
+
+// Err is a structure which contains all information
+// to print a fatal error message in json or pretty mode
+// Err implements error so we can use it anywhere
+type Err struct {
+	msg    string
+	detail string
+	action string
+	hint   string
+}
+
+// Return the error message
+func (u Err) Error() string {
+	if u.detail == "" {
+		return u.msg
+	}
+	return u.detail
+}
+
+// Msg - Replace the current error's message
+func (u Err) Msg(m string, args ...interface{}) Err {
+	return Err{
+		msg:    fmt.Sprintf(m, args...),
+		detail: u.detail,
+		action: u.action,
+		hint:   u.hint,
+	}
+}
+
+// ErrFn function wrapper
+type ErrFn func(err error) Err
+
+// Create a UI error generator, this is needed to simplify
+// the update of the detailed error message in several places
+// in MinIO code
+func newErrFn(msg, action, hint string) ErrFn {
+	return func(err error) Err {
+		u := Err{
+			msg:    msg,
+			action: action,
+			hint:   hint,
+		}
+		if err != nil {
+			u.detail = err.Error()
+		}
+		return u
+	}
+}
+
+// ErrorToErr inspects the passed error and transforms it
+// to the appropriate UI error.
+func ErrorToErr(err error) Err {
+	// If this is already a Err, do nothing
+	if e, ok := err.(Err); ok {
+		return e
+	}
+
+	// Show a generic message for known golang errors
+	if errors.Is(err, syscall.EADDRINUSE) {
+		return ErrPortAlreadyInUse(err).Msg("Specified port is already in use")
+	} else if errors.Is(err, syscall.EACCES) {
+		return ErrPortAccess(err).Msg("Insufficient permissions to use specified port")
+	} else if os.IsPermission(err) {
+		return ErrNoPermissionsToAccessDirFiles(err).Msg("Insufficient permissions to access path")
+	} else if errors.Is(err, io.ErrUnexpectedEOF) {
+		return ErrUnexpectedDataContent(err)
+	} else {
+		// Failed to identify what type of error this, return a simple UI error
+		return Err{msg: err.Error()}
+	}
+
+}
+
+// FmtError converts a fatal error message to a more clear error
+// using some colors
+func FmtError(introMsg string, err error, jsonFlag bool) string {
+	renderedTxt := ""
+	uiErr := ErrorToErr(err)
+	// JSON print
+	if jsonFlag {
+		// Message text in json should be simple
+		if uiErr.detail != "" {
+			return uiErr.msg + ": " + uiErr.detail
+		}
+		return uiErr.msg
+	}
+	// Pretty print error message
+	introMsg += ": "
+	if uiErr.msg != "" {
+		introMsg += color.Bold(uiErr.msg)
+	} else {
+		introMsg += color.Bold(err.Error())
+	}
+	renderedTxt += color.Red(introMsg) + "\n"
+	// Add action message
+	if uiErr.action != "" {
+		renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n"
+	}
+	// Add hint
+	if uiErr.hint != "" {
+		renderedTxt += color.Bold("HINT:") + "\n"
+		renderedTxt += "  " + uiErr.hint
+	}
+	return renderedTxt
+}

cmd/config/errors.go

@@ -0,0 +1,232 @@
+/*
+ * MinIO Cloud Storage, (C) 2018-2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+// UI errors
+var (
+	ErrInvalidConfig = newErrFn(
+		"Invalid value found in the configuration file",
+		"Please ensure a valid value in the configuration file",
+		"For more details, refer to https://docs.min.io/docs/minio-server-configuration-guide",
+	)
+
+	ErrInvalidBrowserValue = newErrFn(
+		"Invalid browser value",
+		"Please check the passed value",
+		"Browser can only accept `on` and `off` values. To disable web browser access, set this value to `off`",
+	)
+
+	ErrInvalidDomainValue = newErrFn(
+		"Invalid domain value",
+		"Please check the passed value",
+		"Domain can only accept DNS compatible values",
+	)
+
+	ErrInvalidErasureSetSize = newErrFn(
+		"Invalid erasure set size",
+		"Please check the passed value",
+		"Erasure set can only accept any of [4, 6, 8, 10, 12, 14, 16] values",
+	)
+
+	ErrInvalidWormValue = newErrFn(
+		"Invalid WORM value",
+		"Please check the passed value",
+		"WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`",
+	)
+
+	ErrInvalidCacheDrivesValue = newErrFn(
+		"Invalid cache drive value",
+		"Please check the value in this ENV variable",
+		"MINIO_CACHE_DRIVES: Mounted drives or directories are delimited by `;`",
+	)
+
+	ErrInvalidCacheExcludesValue = newErrFn(
+		"Invalid cache excludes value",
+		"Please check the passed value",
+		"MINIO_CACHE_EXCLUDE: Cache exclusion patterns are delimited by `;`",
+	)
+
+	ErrInvalidCacheExpiryValue = newErrFn(
+		"Invalid cache expiry value",
+		"Please check the passed value",
+		"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days",
+	)
+
+	ErrInvalidCacheMaxUse = newErrFn(
+		"Invalid cache max-use value",
+		"Please check the passed value",
+		"MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100",
+	)
+
+	ErrInvalidCacheEncryptionKey = newErrFn(
+		"Invalid cache encryption master key value",
+		"Please check the passed value",
+		"MINIO_CACHE_ENCRYPTION_MASTER_KEY: For more information, please refer to https://docs.min.io/docs/minio-disk-cache-guide",
+	)
+
+	ErrInvalidCredentials = newErrFn(
+		"Invalid credentials",
+		"Please provide correct credentials",
+		`Access key length should be between minimum 3 characters in length.
+Secret key should be in between 8 and 40 characters`,
+	)
+
+	ErrEnvCredentialsMissingGateway = newErrFn(
+		"Credentials missing",
+		"Please set your credentials in the environment",
+		`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+	)
+
+	ErrEnvCredentialsMissingDistributed = newErrFn(
+		"Credentials missing",
+		"Please set your credentials in the environment",
+		`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
+	)
+
+	ErrInvalidErasureEndpoints = newErrFn(
+		"Invalid endpoint(s) in erasure mode",
+		"Please provide correct combination of local/remote paths",
+		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
+	)
+
+	ErrInvalidNumberOfErasureEndpoints = newErrFn(
+		"Invalid total number of endpoints for erasure mode",
+		"Please provide an even number of endpoints greater or equal to 4",
+		"For more information, please refer to https://docs.min.io/docs/minio-erasure-code-quickstart-guide",
+	)
+
+	ErrStorageClassValue = newErrFn(
+		"Invalid storage class value",
+		"Please check the value",
+		`MINIO_STORAGE_CLASS_STANDARD: Format "EC:<Default_Parity_Standard_Class>" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request
+MINIO_STORAGE_CLASS_RRS: Format "EC:<Default_Parity_Reduced_Redundancy_Class>" (e.g. "EC:3"). This sets the number of parity disks for MinIO server in Reduced Redundancy mode. Objects are stored in Reduced Redundancy mode, if Put request specifies RRS storage class
+Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`,
+	)
+
+	ErrUnexpectedBackendVersion = newErrFn(
+		"Backend version seems to be too recent",
+		"Please update to the latest MinIO version",
+		"",
+	)
+
+	ErrInvalidAddressFlag = newErrFn(
+		"--address input is invalid",
+		"Please check --address parameter",
+		`--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000')
+	Examples: --address ':443'
+		  --address '172.16.34.31:9000'
+		  --address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`,
+	)
+
+	ErrInvalidFSEndpoint = newErrFn(
+		"Invalid endpoint for standalone FS mode",
+		"Please check the FS endpoint",
+		`FS mode requires only one writable disk path
+Example 1:
+   $ minio server /data/minio/`,
+	)
+
+	ErrUnableToWriteInBackend = newErrFn(
+		"Unable to write to the backend",
+		"Please ensure MinIO binary has write permissions for the backend",
+		`Verify if MinIO binary is running as the same user who has write permissions for the backend`,
+	)
+
+	ErrPortAlreadyInUse = newErrFn(
+		"Port is already in use",
+		"Please ensure no other program uses the same address/port",
+		"",
+	)
+
+	ErrPortAccess = newErrFn(
+		"Unable to use specified port",
+		"Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions",
+		`Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`,
+	)
+
+	ErrNoPermissionsToAccessDirFiles = newErrFn(
+		"Missing permissions to access the specified path",
+		"Please ensure the specified path can be accessed",
+		"",
+	)
+
+	ErrSSLUnexpectedError = newErrFn(
+		"Invalid TLS certificate",
+		"Please check the content of your certificate data",
+		`Only PEM (x.509) format is accepted as valid public & private certificates`,
+	)
+
+	ErrSSLUnexpectedData = newErrFn(
+		"Invalid TLS certificate",
+		"Please check your certificate",
+		"",
+	)
+
+	ErrSSLNoPassword = newErrFn(
+		"Missing TLS password",
+		"Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted",
+		"",
+	)
+
+	ErrNoCertsAndHTTPSEndpoints = newErrFn(
+		"HTTPS specified in endpoints, but no TLS certificate is found on the local machine",
+		"Please add TLS certificate or use HTTP endpoints only",
+		"Refer to https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls for information about how to load a TLS certificate in your server",
+	)
+
+	ErrCertsAndHTTPEndpoints = newErrFn(
+		"HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate",
+		"Please remove the certificate in the configuration directory or switch to HTTPS",
+		"",
+	)
+
+	ErrSSLWrongPassword = newErrFn(
+		"Unable to decrypt the private key using the provided password",
+		"Please set the correct password in environment variable `MINIO_CERT_PASSWD`",
+		"",
+	)
+
+	ErrUnexpectedDataContent = newErrFn(
+		"Unexpected data content",
+		"Please contact MinIO at https://slack.min.io",
+		"",
+	)
+
+	ErrUnexpectedError = newErrFn(
+		"Unexpected error",
+		"Please contact MinIO at https://slack.min.io",
+		"",
+	)
+
+	ErrInvalidCompressionIncludesValue = newErrFn(
+		"Invalid compression include value",
+		"Please check the passed value",
+		"Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_ATTR=\"A,B,C\"",
+	)
+
+	ErrInvalidGWSSEValue = newErrFn(
+		"Invalid gateway SSE value",
+		"Please check the passed value",
+		"MINIO_GATEWAY_SSE: Gateway SSE accepts only C and S3 as valid values. Delimit by `;` to set more than one value",
+	)
+
+	ErrInvalidGWSSEEnvValue = newErrFn(
+		"Invalid gateway SSE configuration",
+		"",
+		"Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE",
+	)
+)

cmd/config/ldap/config.go

@@ -0,0 +1,201 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ldap
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"regexp"
+	"time"
+
+	"github.com/minio/minio/pkg/env"
+	ldap "gopkg.in/ldap.v3"
+)
+
+const (
+	defaultLDAPExpiry = time.Hour * 1
+)
+
+// Config contains AD/LDAP server connectivity information.
+type Config struct {
+	IsEnabled bool `json:"enabled"`
+
+	// E.g. "ldap.minio.io:636"
+	ServerAddr string `json:"serverAddr"`
+
+	// STS credentials expiry duration
+	STSExpiryDuration string        `json:"stsExpiryDuration"`
+	stsExpiryDuration time.Duration // contains converted value
+
+	RootCAs *x509.CertPool `json:"-"`
+
+	// Format string for usernames
+	UsernameFormat string `json:"usernameFormat"`
+
+	GroupSearchBaseDN  string `json:"groupSearchBaseDN"`
+	GroupSearchFilter  string `json:"groupSearchFilter"`
+	GroupNameAttribute string `json:"groupNameAttribute"`
+}
+
+// LDAP keys and envs.
+const (
+	ServerAddr         = "server_addr"
+	STSExpiry          = "sts_expiry"
+	UsernameFormat     = "username_format"
+	GroupSearchFilter  = "group_search_filter"
+	GroupNameAttribute = "group_name_attribute"
+	GroupSearchBaseDN  = "group_search_base_dn"
+
+	EnvServerAddr         = "MINIO_IDENTITY_LDAP_SERVER_ADDR"
+	EnvSTSExpiry          = "MINIO_IDENTITY_LDAP_STS_EXPIRY"
+	EnvUsernameFormat     = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT"
+	EnvGroupSearchFilter  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
+	EnvGroupNameAttribute = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
+	EnvGroupSearchBaseDN  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
+)
+
+// Connect connect to ldap server.
+func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
+	if l == nil {
+		// Happens when LDAP is not configured.
+		return
+	}
+	return ldap.DialTLS("tcp", l.ServerAddr, &tls.Config{RootCAs: l.RootCAs})
+}
+
+// GetExpiryDuration - return parsed expiry duration.
+func (l Config) GetExpiryDuration() time.Duration {
+	return l.stsExpiryDuration
+}
+
+// Lookup - initializes LDAP config, overrides config, if any ENV values are set.
+func Lookup(cfg Config, rootCAs *x509.CertPool) (l Config, err error) {
+	if cfg.ServerAddr == "" && cfg.IsEnabled {
+		return l, errors.New("ldap server cannot initialize with empty LDAP server")
+	}
+	l.RootCAs = rootCAs
+	ldapServer := env.Get(EnvServerAddr, cfg.ServerAddr)
+	if ldapServer == "" {
+		return l, nil
+	}
+	l.IsEnabled = true
+	l.ServerAddr = ldapServer
+	l.stsExpiryDuration = defaultLDAPExpiry
+	if v := env.Get(EnvSTSExpiry, cfg.STSExpiryDuration); v != "" {
+		expDur, err := time.ParseDuration(v)
+		if err != nil {
+			return l, errors.New("LDAP expiry time err:" + err.Error())
+		}
+		if expDur <= 0 {
+			return l, errors.New("LDAP expiry time has to be positive")
+		}
+		l.STSExpiryDuration = v
+		l.stsExpiryDuration = expDur
+	}
+
+	if v := env.Get(EnvUsernameFormat, cfg.UsernameFormat); v != "" {
+		subs, err := NewSubstituter("username", "test")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(v); err != nil {
+			return l, fmt.Errorf("Only username may be substituted in the username format: %s", err)
+		}
+		l.UsernameFormat = v
+	}
+
+	grpSearchFilter := env.Get(EnvGroupSearchFilter, cfg.GroupSearchFilter)
+	grpSearchNameAttr := env.Get(EnvGroupNameAttribute, cfg.GroupNameAttribute)
+	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, cfg.GroupSearchBaseDN)
+
+	// Either all group params must be set or none must be set.
+	allNotSet := grpSearchFilter == "" && grpSearchNameAttr == "" && grpSearchBaseDN == ""
+	allSet := grpSearchFilter != "" && grpSearchNameAttr != "" && grpSearchBaseDN != ""
+	if !allNotSet && !allSet {
+		return l, errors.New("All group related parameters must be set")
+	}
+
+	if allSet {
+		subs, err := NewSubstituter("username", "test", "usernamedn", "test2")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(grpSearchFilter); err != nil {
+			return l, fmt.Errorf("Only username and usernamedn may be substituted in the group search filter string: %s", err)
+		}
+		l.GroupSearchFilter = grpSearchFilter
+		l.GroupNameAttribute = grpSearchNameAttr
+		subs, err = NewSubstituter("username", "test", "usernamedn", "test2")
+		if err != nil {
+			return l, err
+		}
+		if _, err := subs.Substitute(grpSearchBaseDN); err != nil {
+			return l, fmt.Errorf("Only username and usernamedn may be substituted in the base DN string: %s", err)
+		}
+		l.GroupSearchBaseDN = grpSearchBaseDN
+	}
+	return
+}
+
+// Substituter - This type is to allow restricted runtime
+// substitutions of variables in LDAP configuration items during
+// runtime.
+type Substituter struct {
+	vals map[string]string
+}
+
+// NewSubstituter - sets up the substituter for usage, for e.g.:
+//
+//  subber := NewSubstituter("username", "john")
+func NewSubstituter(v ...string) (Substituter, error) {
+	if len(v)%2 != 0 {
+		return Substituter{}, errors.New("Need an even number of arguments")
+	}
+	vals := make(map[string]string)
+	for i := 0; i < len(v); i += 2 {
+		vals[v[i]] = v[i+1]
+	}
+	return Substituter{vals: vals}, nil
+}
+
+// Substitute - performs substitution on the given string `t`. Returns
+// an error if there are any variables in the input that do not have
+// values in the substituter. E.g.:
+//
+// subber.Substitute("uid=${username},cn=users,dc=example,dc=com")
+//
+// returns "uid=john,cn=users,dc=example,dc=com"
+//
+// whereas:
+//
+// subber.Substitute("uid=${usernamedn}")
+//
+// returns an error.
+func (s *Substituter) Substitute(t string) (string, error) {
+	for k, v := range s.vals {
+		re := regexp.MustCompile(fmt.Sprintf(`\$\{%s\}`, k))
+		t = re.ReplaceAllLiteralString(t, v)
+	}
+	// Check if all requested substitutions have been made.
+	re := regexp.MustCompile(`\$\{.*\}`)
+	if re.MatchString(t) {
+		return "", errors.New("unsupported substitution requested")
+	}
+	return t, nil
+}

cmd/config/ldap/config_test.go

@@ -0,0 +1,65 @@
+/*
+ * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ldap
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestSubstituter(t *testing.T) {
+	tests := []struct {
+		KV               []string
+		SubstitutableStr string
+		SubstitutedStr   string
+		ErrExpected      bool
+	}{
+		{
+			KV:               []string{"username", "john"},
+			SubstitutableStr: "uid=${username},cn=users,dc=example,dc=com",
+			SubstitutedStr:   "uid=john,cn=users,dc=example,dc=com",
+			ErrExpected:      false,
+		},
+		{
+			KV:               []string{"username", "john"},
+			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
+			ErrExpected:      true,
+		},
+		{
+			KV:               []string{"username"},
+			SubstitutableStr: "uid=${usernamedn},cn=users,dc=example,dc=com",
+			ErrExpected:      true,
+		},
+	}
+
+	for i, test := range tests {
+		test := test
+		t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) {
+			subber, err := NewSubstituter(test.KV...)
+			if err != nil && !test.ErrExpected {
+				t.Errorf("Unexpected failure %s", err)
+			}
+			gotStr, err := subber.Substitute(test.SubstitutableStr)
+			if err != nil && !test.ErrExpected {
+				t.Errorf("Unexpected failure %s", err)
+			}
+			if gotStr != test.SubstitutedStr {
+				t.Errorf("Expected %s, got %s", test.SubstitutedStr, gotStr)
+			}
+		})
+	}
+}