mirror of
https://github.com/pgsty/minio.git
synced 2026-07-19 12:10:24 +03:00
Reduce number of envs and options from command line. (#3230)
Ref #3229 After review with @abperiasamy we decided to remove all the unnecessary options - MINIO_BROWSER (Implemented as a security feature but now deemed obsolete since even if blocking access to MINIO_BROWSER, s3 API port is open) - MINIO_CACHE_EXPIRY (Defaults to 72h) - MINIO_MAXCONN (No one used this option and we don't test this) - MINIO_ENABLE_FSMETA (Enable FSMETA all the time) Remove --ignore-disks option - this option was implemented when XL layer would initialize the backend disks and heal them automatically to disallow XL accidentally using the root partition itself this option was introduced. This behavior has been changed XL no longer automatically initializes `format.json` a HEAL is controlled activity, so ignore-disks is not useful anymore. This change also addresses the problems of our documentation going forward and keeps things simple. This patch brings in reduction of options and defaulting them to a valid known inputs. This patch also serves as a guideline of limiting many ways to do the same thing.
This commit is contained in:
+2
-13
@@ -16,10 +16,7 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/urfave/negroni"
|
||||
)
|
||||
import router "github.com/gorilla/mux"
|
||||
|
||||
// objectAPIHandler implements and provides http handlers for S3 API.
|
||||
type objectAPIHandlers struct {
|
||||
@@ -34,7 +31,7 @@ func registerAPIRouter(mux *router.Router) {
|
||||
}
|
||||
|
||||
// API Router
|
||||
apiRouter := mux.NewRoute().PathPrefix("").Subrouter()
|
||||
apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
|
||||
|
||||
// Bucket router
|
||||
bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
|
||||
@@ -99,12 +96,4 @@ func registerAPIRouter(mux *router.Router) {
|
||||
|
||||
// ListBuckets
|
||||
apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
|
||||
|
||||
mux.PathPrefix("/").Handler(negroni.New(
|
||||
// Validates all incoming requests to have a valid date header.
|
||||
negroni.Wrap(timeValidityHandler{}),
|
||||
// Route requests
|
||||
negroni.Wrap(apiRouter),
|
||||
))
|
||||
|
||||
}
|
||||
|
||||
@@ -51,7 +51,7 @@ func prepareBenchmarkBackend(instanceType string) (ObjectLayer, []string, error)
|
||||
return nil, nil, err
|
||||
}
|
||||
// initialize object layer.
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
@@ -226,7 +226,7 @@ func TestErasureReadUtils(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(endpoints, nil)
|
||||
objLayer, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
removeRoots(disks)
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -43,7 +43,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -97,7 +97,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints, nil)
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -128,7 +128,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints, nil)
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -159,7 +159,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints, nil)
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
@@ -280,7 +280,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -316,7 +316,7 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -605,7 +605,7 @@ func TestInitFormatXLErrors(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -711,7 +711,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -741,7 +741,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -769,7 +769,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -795,7 +795,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -824,7 +824,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -846,7 +846,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -870,7 +870,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -896,7 +896,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -918,7 +918,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -944,7 +944,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Corrupted format json in one disk
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -974,7 +974,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -995,7 +995,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1019,7 +1019,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1045,7 +1045,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1067,7 +1067,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1093,7 +1093,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -18,10 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -164,32 +162,3 @@ func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) b
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var extendedHeaders = []string{
|
||||
"X-Amz-Meta-",
|
||||
"X-Minio-Meta-",
|
||||
// Add new extended headers.
|
||||
}
|
||||
|
||||
// isExtendedHeader validates if input string matches extended headers.
|
||||
func isExtendedHeader(header string) bool {
|
||||
for _, extendedHeader := range extendedHeaders {
|
||||
if strings.HasPrefix(header, extendedHeader) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Return true if extended HTTP headers are set, false otherwise.
|
||||
func hasExtendedHeader(metadata map[string]string) bool {
|
||||
if os.Getenv("MINIO_ENABLE_FSMETA") == "1" {
|
||||
return true
|
||||
}
|
||||
for k := range metadata {
|
||||
if isExtendedHeader(k) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -23,56 +23,12 @@ import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests scenarios which can occur for hasExtendedHeader function.
|
||||
func TestHasExtendedHeader(t *testing.T) {
|
||||
// All test cases concerning hasExtendedHeader function.
|
||||
testCases := []struct {
|
||||
metadata map[string]string
|
||||
has bool
|
||||
}{
|
||||
// Verifies if X-Amz-Meta is present.
|
||||
{
|
||||
metadata: map[string]string{
|
||||
"X-Amz-Meta-1": "value",
|
||||
},
|
||||
has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
|
||||
},
|
||||
// Verifies if X-Minio-Meta is present.
|
||||
{
|
||||
metadata: map[string]string{
|
||||
"X-Minio-Meta-1": "value",
|
||||
},
|
||||
has: true || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
|
||||
},
|
||||
// Verifies if extended header is not present.
|
||||
{
|
||||
metadata: map[string]string{
|
||||
"md5Sum": "value",
|
||||
},
|
||||
has: false || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
|
||||
},
|
||||
// Verifies if extended header is not present, but with an empty input.
|
||||
{
|
||||
metadata: nil,
|
||||
has: false || os.Getenv("MINIO_ENABLE_FSMETA") == "1",
|
||||
},
|
||||
}
|
||||
|
||||
// Validate all test cases.
|
||||
for i, testCase := range testCases {
|
||||
has := hasExtendedHeader(testCase.metadata)
|
||||
if has != testCase.has {
|
||||
t.Fatalf("Test case %d: Expected \"%#v\", but got \"%#v\"", i+1, testCase.has, has)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) {
|
||||
endpoints, err := parseStorageEndpoints([]string{disk})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints, nil)
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
+11
-15
@@ -227,10 +227,8 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
|
||||
// Initialize `fs.json` values.
|
||||
fsMeta := newFSMetaV1()
|
||||
|
||||
// Save additional metadata only if extended headers such as "X-Amz-Meta-" are set.
|
||||
if hasExtendedHeader(meta) {
|
||||
fsMeta.Meta = meta
|
||||
}
|
||||
// Save additional metadata.
|
||||
fsMeta.Meta = meta
|
||||
|
||||
// This lock needs to be held for any changes to the directory
|
||||
// contents of ".minio.sys/multipart/object/"
|
||||
@@ -765,18 +763,16 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
|
||||
// No need to save part info, since we have concatenated all parts.
|
||||
fsMeta.Parts = nil
|
||||
|
||||
// Save additional metadata only if extended headers such as "X-Amz-Meta-" are set.
|
||||
if hasExtendedHeader(fsMeta.Meta) {
|
||||
if len(fsMeta.Meta) == 0 {
|
||||
fsMeta.Meta = make(map[string]string)
|
||||
}
|
||||
fsMeta.Meta["md5Sum"] = s3MD5
|
||||
// Save additional metadata.
|
||||
if len(fsMeta.Meta) == 0 {
|
||||
fsMeta.Meta = make(map[string]string)
|
||||
}
|
||||
fsMeta.Meta["md5Sum"] = s3MD5
|
||||
|
||||
fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
// Write the metadata to a temp file and rename it to the actual location.
|
||||
if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
|
||||
return "", toObjectErr(err, bucket, object)
|
||||
}
|
||||
fsMetaPath = path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
// Write the metadata to a temp file and rename it to the actual location.
|
||||
if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
|
||||
return "", toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
// Cleanup all the parts if everything else has been safely committed.
|
||||
|
||||
+7
-9
@@ -455,17 +455,15 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
|
||||
return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// Save additional metadata only if extended headers such as "X-Amz-Meta-" are set.
|
||||
if hasExtendedHeader(metadata) {
|
||||
// Initialize `fs.json` values.
|
||||
fsMeta := newFSMetaV1()
|
||||
fsMeta.Meta = metadata
|
||||
// Save additional metadata. Initialize `fs.json` values.
|
||||
fsMeta := newFSMetaV1()
|
||||
fsMeta.Meta = metadata
|
||||
|
||||
fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
|
||||
}
|
||||
fsMetaPath := path.Join(bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
if err = writeFSMetadata(fs.storage, minioMetaBucket, fsMetaPath, fsMeta); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
objInfo, err = fs.getObjectInfo(bucket, object)
|
||||
if err == nil {
|
||||
// If MINIO_ENABLE_FSMETA is not enabled objInfo.MD5Sum will be empty.
|
||||
|
||||
+12
-6
@@ -45,7 +45,7 @@ func TestNewFS(t *testing.T) {
|
||||
t.Fatal("Uexpected error: ", err)
|
||||
}
|
||||
|
||||
fsStorageDisks, err := initStorageDisks(endpoints, nil)
|
||||
fsStorageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Uexpected error: ", err)
|
||||
}
|
||||
@@ -55,7 +55,7 @@ func TestNewFS(t *testing.T) {
|
||||
t.Fatal("Uexpected error: ", err)
|
||||
}
|
||||
|
||||
xlStorageDisks, err := initStorageDisks(endpoints, nil)
|
||||
xlStorageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Uexpected error: ", err)
|
||||
}
|
||||
@@ -96,17 +96,22 @@ func TestNewFS(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSShutdown - initialize a new FS object layer then calls Shutdown
|
||||
// to check returned results
|
||||
// TestFSShutdown - initialize a new FS object layer then calls
|
||||
// Shutdown to check returned results
|
||||
func TestFSShutdown(t *testing.T) {
|
||||
rootPath, err := newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
|
||||
bucketName := "testbucket"
|
||||
objectName := "object"
|
||||
// Create and return an fsObject with its path in the disk
|
||||
prepareTest := func() (fsObjects, string) {
|
||||
disk := filepath.Join(os.TempDir(), "minio-"+nextSuffix())
|
||||
obj := initFSObjects(disk, t)
|
||||
fs := obj.(fsObjects)
|
||||
bucketName := "testbucket"
|
||||
objectName := "object"
|
||||
objectContent := "12345"
|
||||
obj.MakeBucket(bucketName)
|
||||
sha256sum := ""
|
||||
@@ -124,6 +129,7 @@ func TestFSShutdown(t *testing.T) {
|
||||
// Test Shutdown with faulty disk
|
||||
for i := 1; i <= 5; i++ {
|
||||
fs, disk := prepareTest()
|
||||
fs.DeleteObject(bucketName, objectName)
|
||||
fsStorage := fs.storage.(*posix)
|
||||
fs.storage = newNaughtyDisk(fsStorage, map[int]error{i: errFaultyDisk}, nil)
|
||||
if err := fs.Shutdown(); errorCause(err) != errFaultyDisk {
|
||||
|
||||
+28
-23
@@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
@@ -77,27 +76,24 @@ func setBrowserRedirectHandler(h http.Handler) http.Handler {
|
||||
}
|
||||
|
||||
func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") {
|
||||
// Re-direction handled specifically for browsers.
|
||||
if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) {
|
||||
switch r.URL.Path {
|
||||
case "/", "/webrpc", "/login", "/favicon.ico":
|
||||
// '/' is redirected to 'locationPrefix/'
|
||||
// '/webrpc' is redirected to 'locationPrefix/webrpc'
|
||||
// '/login' is redirected to 'locationPrefix/login'
|
||||
location := h.locationPrefix + r.URL.Path
|
||||
// Redirect to new location.
|
||||
http.Redirect(w, r, location, http.StatusTemporaryRedirect)
|
||||
return
|
||||
case h.locationPrefix:
|
||||
// locationPrefix is redirected to 'locationPrefix/'
|
||||
location := h.locationPrefix + "/"
|
||||
http.Redirect(w, r, location, http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
// Re-direction handled specifically for browsers.
|
||||
if strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && !isRequestSignatureV4(r) {
|
||||
switch r.URL.Path {
|
||||
case "/", "/webrpc", "/login", "/favicon.ico":
|
||||
// '/' is redirected to 'locationPrefix/'
|
||||
// '/webrpc' is redirected to 'locationPrefix/webrpc'
|
||||
// '/login' is redirected to 'locationPrefix/login'
|
||||
location := h.locationPrefix + r.URL.Path
|
||||
// Redirect to new location.
|
||||
http.Redirect(w, r, location, http.StatusTemporaryRedirect)
|
||||
return
|
||||
case h.locationPrefix:
|
||||
// locationPrefix is redirected to 'locationPrefix/'
|
||||
location := h.locationPrefix + "/"
|
||||
http.Redirect(w, r, location, http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
h.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
@@ -187,11 +183,19 @@ func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) {
|
||||
return time.Time{}, ErrMissingDateHeader
|
||||
}
|
||||
|
||||
type timeValidityHandler struct{}
|
||||
type timeValidityHandler struct {
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
// setTimeValidityHandler to validate parsable time over http header
|
||||
func setTimeValidityHandler(h http.Handler) http.Handler {
|
||||
return timeValidityHandler{h}
|
||||
}
|
||||
|
||||
func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Verify if date headers are set, if not reject the request
|
||||
if _, ok := r.Header["Authorization"]; ok {
|
||||
aType := getRequestAuthType(r)
|
||||
if aType != authTypeAnonymous && aType != authTypeJWT {
|
||||
// Verify if date headers are set, if not reject the request
|
||||
amzDate, apiErr := parseAmzDateHeader(r)
|
||||
if apiErr != ErrNone {
|
||||
// All our internal APIs are sensitive towards Date
|
||||
@@ -208,6 +212,7 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
h.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
type resourceHandler struct {
|
||||
|
||||
+1
-4
@@ -41,14 +41,11 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
globalQuiet = false // Quiet flag set via command line
|
||||
globalQuiet = false // Quiet flag set via command line.
|
||||
globalIsDistXL = false // "Is Distributed?" flag.
|
||||
|
||||
// Add new global flags here.
|
||||
|
||||
// Maximum connections handled per
|
||||
// server, defaults to 0 (unlimited).
|
||||
globalMaxConn = 0
|
||||
// Maximum cache size.
|
||||
globalMaxCacheSize = uint64(maxCacheSize)
|
||||
// Cache expiry.
|
||||
|
||||
@@ -83,11 +83,6 @@ func registerDistNSLockRouter(mux *router.Router, serverConfig serverCmdConfig)
|
||||
// Create one lock server for every local storage rpc server.
|
||||
func newLockServers(srvConfig serverCmdConfig) (lockServers []*lockServer) {
|
||||
for _, ep := range srvConfig.endpoints {
|
||||
if containsEndpoint(srvConfig.ignoredEndpoints, ep) {
|
||||
// Skip initializing ignored endpoint.
|
||||
continue
|
||||
}
|
||||
|
||||
// Not local storage move to the next node.
|
||||
if !isLocalStorage(ep) {
|
||||
continue
|
||||
|
||||
@@ -475,7 +475,7 @@ func TestLockServers(t *testing.T) {
|
||||
},
|
||||
totalLockServers: 1,
|
||||
},
|
||||
// Test - 2 two servers possible, 1 ignored.
|
||||
// Test - 2 two servers possible.
|
||||
{
|
||||
isDistXL: true,
|
||||
srvCmdConfig: serverCmdConfig{
|
||||
@@ -496,13 +496,8 @@ func TestLockServers(t *testing.T) {
|
||||
Host: "1.1.2.2:9000",
|
||||
Path: "/mnt/disk4",
|
||||
}},
|
||||
ignoredEndpoints: []*url.URL{{
|
||||
Scheme: "http",
|
||||
Host: "localhost:9000",
|
||||
Path: "/mnt/disk2",
|
||||
}},
|
||||
},
|
||||
totalLockServers: 1,
|
||||
totalLockServers: 2,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -42,7 +42,7 @@ var (
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "quiet",
|
||||
Usage: "Suppress chatty output.",
|
||||
Usage: "Disable startup information.",
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -218,7 +218,7 @@ func Main() {
|
||||
}
|
||||
|
||||
// Start profiler if env is set.
|
||||
if profiler := os.Getenv("MINIO_PROFILER"); profiler != "" {
|
||||
if profiler := os.Getenv("_MINIO_PROFILER"); profiler != "" {
|
||||
globalProfiler = startProfiler(profiler)
|
||||
}
|
||||
|
||||
|
||||
@@ -566,13 +566,12 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
|
||||
// Initialize FS backend for the benchmark.
|
||||
func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) {
|
||||
|
||||
endPoints, err := parseStorageEndpoints([]string{disk})
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
obj, _, err = initObjectLayer(endPoints, nil)
|
||||
obj, _, err = initObjectLayer(endPoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
@@ -45,7 +45,11 @@ type printOnceFunc func(msg string)
|
||||
func printOnceFn() printOnceFunc {
|
||||
var once sync.Once
|
||||
return func(msg string) {
|
||||
once.Do(func() { console.Println(msg) })
|
||||
once.Do(func() {
|
||||
if !globalQuiet {
|
||||
console.Println(msg)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+1
-14
@@ -266,7 +266,7 @@ func retryFormattingDisks(firstDisk bool, endpoints []*url.URL, storageDisks []S
|
||||
}
|
||||
|
||||
// Initialize storage disks based on input arguments.
|
||||
func initStorageDisks(endpoints, ignoredEndpoints []*url.URL) ([]StorageAPI, error) {
|
||||
func initStorageDisks(endpoints []*url.URL) ([]StorageAPI, error) {
|
||||
// Single disk means we will use FS backend.
|
||||
if len(endpoints) == 1 {
|
||||
if endpoints[0] == nil {
|
||||
@@ -284,19 +284,6 @@ func initStorageDisks(endpoints, ignoredEndpoints []*url.URL) ([]StorageAPI, err
|
||||
if ep == nil {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
// Check if disk is ignored.
|
||||
ignored := false
|
||||
for _, iep := range ignoredEndpoints {
|
||||
if *ep == *iep {
|
||||
ignored = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if ignored {
|
||||
// Set this situation as disk not found.
|
||||
storageDisks[index] = nil
|
||||
continue
|
||||
}
|
||||
// Intentionally ignore disk not found errors. XL is designed
|
||||
// to handle these errors internally.
|
||||
storage, err := newStorageAPI(ep)
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
var errTooManyRequests = errors.New("Too many clients in the waiting list")
|
||||
|
||||
// rateLimit - represents datatype of the functionality implemented to
|
||||
// limit the number of concurrent http requests.
|
||||
type rateLimit struct {
|
||||
handler http.Handler
|
||||
workQueue chan struct{}
|
||||
waitQueue chan struct{}
|
||||
}
|
||||
|
||||
// acquire and release implement a way to send and receive from the
|
||||
// channel this is in-turn used to rate limit incoming connections in
|
||||
// ServeHTTP() http.Handler method.
|
||||
func (c *rateLimit) acquire() error {
|
||||
// attempt to enter the waitQueue. If no slot is immediately
|
||||
// available return error.
|
||||
select {
|
||||
case c.waitQueue <- struct{}{}:
|
||||
// entered wait queue
|
||||
break
|
||||
default:
|
||||
// no slot available for waiting
|
||||
return errTooManyRequests
|
||||
}
|
||||
|
||||
// block attempting to enter the workQueue. If the workQueue
|
||||
// is full, there can be at most cap(waitQueue) ==
|
||||
// 4*globalMaxConn goroutines waiting here because of the
|
||||
// select above.
|
||||
select {
|
||||
case c.workQueue <- struct{}{}:
|
||||
// entered workQueue - so remove one waiter. This step
|
||||
// does not block as the waitQueue cannot be empty.
|
||||
<-c.waitQueue
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Release one element from workQueue to serve a new client in the
|
||||
// waiting list
|
||||
func (c *rateLimit) release() {
|
||||
<-c.workQueue
|
||||
}
|
||||
|
||||
// ServeHTTP is an http.Handler ServeHTTP method, implemented to rate
|
||||
// limit incoming HTTP requests.
|
||||
func (c *rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Acquire the connection if queue is not full, otherwise
|
||||
// code path waits here until the previous case is true.
|
||||
if err := c.acquire(); err != nil {
|
||||
w.WriteHeader(http.StatusTooManyRequests)
|
||||
return
|
||||
}
|
||||
|
||||
// Serves the request.
|
||||
c.handler.ServeHTTP(w, r)
|
||||
|
||||
// Release
|
||||
c.release()
|
||||
}
|
||||
|
||||
// setRateLimitHandler limits the number of concurrent http requests
|
||||
// based on MINIO_MAXCONN.
|
||||
func setRateLimitHandler(handler http.Handler) http.Handler {
|
||||
if globalMaxConn == 0 {
|
||||
return handler
|
||||
} // else proceed to rate limiting.
|
||||
|
||||
// For max connection limit of > '0' we initialize rate limit
|
||||
// handler.
|
||||
return &rateLimit{
|
||||
handler: handler,
|
||||
workQueue: make(chan struct{}, globalMaxConn),
|
||||
waitQueue: make(chan struct{}, globalMaxConn*4),
|
||||
}
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// This test sets globalMaxConn to 1 and starts 6 connections in
|
||||
// parallel on a server with the rate limit handler configured. This
|
||||
// should allow one request to execute at a time, and at most 4 to
|
||||
// wait to execute and the 6th request should get a 429 status code
|
||||
// error.
|
||||
func TestRateLimitHandler(t *testing.T) {
|
||||
// save the global Max connections
|
||||
saveGlobalMaxConn := globalMaxConn
|
||||
|
||||
globalMaxConn = 1
|
||||
testHandler := func(w http.ResponseWriter, r *http.Request) {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
fmt.Fprintln(w, "Hello client!")
|
||||
}
|
||||
rlh := setRateLimitHandler(http.HandlerFunc(testHandler))
|
||||
ts := httptest.NewServer(rlh)
|
||||
respCh := make(chan int)
|
||||
startTime := time.Now()
|
||||
for i := 0; i < 6; i++ {
|
||||
go func(ch chan<- int) {
|
||||
resp, err := http.Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Errorf(
|
||||
"Got error requesting test server - %v\n",
|
||||
err,
|
||||
)
|
||||
}
|
||||
respCh <- resp.StatusCode
|
||||
}(respCh)
|
||||
}
|
||||
|
||||
tooManyReqErrCount := 0
|
||||
for i := 0; i < 6; i++ {
|
||||
code := <-respCh
|
||||
if code == 429 {
|
||||
tooManyReqErrCount++
|
||||
} else if code != 200 {
|
||||
t.Errorf("Got non-200 resp code - %d\n", code)
|
||||
}
|
||||
}
|
||||
duration := time.Since(startTime)
|
||||
if duration < time.Duration(500*time.Millisecond) {
|
||||
// as globalMaxConn is 1, only 1 request will execute
|
||||
// at a time, and the five allowed requested will take
|
||||
// at least 500 ms.
|
||||
t.Errorf("Expected all requests to take at least 500ms, but it was done in %v\n",
|
||||
duration)
|
||||
}
|
||||
if tooManyReqErrCount != 1 {
|
||||
t.Errorf("Expected to get 1 error, but got %d",
|
||||
tooManyReqErrCount)
|
||||
}
|
||||
ts.Close()
|
||||
|
||||
// restore the global Max connections
|
||||
globalMaxConn = saveGlobalMaxConn
|
||||
}
|
||||
+8
-14
@@ -18,8 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
)
|
||||
@@ -109,17 +107,13 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// set environmental variable MINIO_BROWSER=off to disable minio web browser.
|
||||
// By default minio web browser is enabled.
|
||||
if !strings.EqualFold(os.Getenv("MINIO_BROWSER"), "off") {
|
||||
// Register RPC router for web related calls.
|
||||
if err = registerBrowserPeerRPCRouter(mux); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Register RPC router for web related calls.
|
||||
if err = registerBrowserPeerRPCRouter(mux); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = registerWebRouter(mux); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = registerWebRouter(mux); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add API router.
|
||||
@@ -127,8 +121,6 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error)
|
||||
|
||||
// List of some generic handlers which are applied for all incoming requests.
|
||||
var handlerFns = []HandlerFunc{
|
||||
// Limits the number of concurrent http requests.
|
||||
setRateLimitHandler,
|
||||
// Limits all requests size to a maximum fixed limit
|
||||
setRequestSizeLimitHandler,
|
||||
// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
|
||||
@@ -139,6 +131,8 @@ func configureServerHandler(srvCmdConfig serverCmdConfig) (http.Handler, error)
|
||||
setPrivateBucketHandler,
|
||||
// Adds cache control for all browser requests.
|
||||
setBrowserCacheControlHandler,
|
||||
// Validates all incoming requests to have a valid date header.
|
||||
setTimeValidityHandler,
|
||||
// CORS setting for all browser API requests.
|
||||
setCorsHandler,
|
||||
// Validates all incoming URL resources, for invalid/unsupported
|
||||
|
||||
+7
-70
@@ -22,9 +22,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"regexp"
|
||||
"runtime"
|
||||
@@ -38,10 +36,6 @@ var serverFlags = []cli.Flag{
|
||||
Value: ":9000",
|
||||
Usage: `Bind to a specific IP:PORT. Defaults to ":9000".`,
|
||||
},
|
||||
cli.StringFlag{
|
||||
Name: "ignore-disks",
|
||||
Usage: `Comma separated list of faulty drives to ignore at startup.`,
|
||||
},
|
||||
}
|
||||
|
||||
var serverCmd = cli.Command{
|
||||
@@ -63,13 +57,6 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_ACCESS_KEY: Username or access key of 5 to 20 characters in length.
|
||||
MINIO_SECRET_KEY: Password or secret key of 8 to 40 characters in length.
|
||||
|
||||
CACHING:
|
||||
MINIO_CACHE_SIZE: Limit maximum cache size. Allowed units are [GB|MB|KB]. Defaults to 8GB.
|
||||
MINIO_CACHE_EXPIRY: Automatically expire cached objects. Allowed units are [h|m|s]. Defaults to 72h.
|
||||
|
||||
SECURITY:
|
||||
MINIO_SECURE_CONSOLE: Set secure console to 'no' to disable printing secret key. Defaults to 'yes'.
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio server on "/home/shared" directory.
|
||||
$ minio {{.Name}} /home/shared
|
||||
@@ -92,10 +79,9 @@ EXAMPLES:
|
||||
}
|
||||
|
||||
type serverCmdConfig struct {
|
||||
serverAddr string
|
||||
endpoints []*url.URL
|
||||
ignoredEndpoints []*url.URL
|
||||
storageDisks []StorageAPI
|
||||
serverAddr string
|
||||
endpoints []*url.URL
|
||||
storageDisks []StorageAPI
|
||||
}
|
||||
|
||||
// Parse an array of end-points (from the command line)
|
||||
@@ -190,27 +176,6 @@ func initServerConfig(c *cli.Context) {
|
||||
err := createCertsPath()
|
||||
fatalIf(err, "Unable to create \"certs\" directory.")
|
||||
|
||||
// Fetch max conn limit from environment variable.
|
||||
if maxConnStr := os.Getenv("MINIO_MAXCONN"); maxConnStr != "" {
|
||||
// We need to parse to its integer value.
|
||||
globalMaxConn, err = strconv.Atoi(maxConnStr)
|
||||
fatalIf(err, "Unable to convert MINIO_MAXCONN=%s environment variable into its integer value.", maxConnStr)
|
||||
}
|
||||
|
||||
// Fetch max cache size from environment variable.
|
||||
if maxCacheSizeStr := os.Getenv("MINIO_CACHE_SIZE"); maxCacheSizeStr != "" {
|
||||
// We need to parse cache size to its integer value.
|
||||
globalMaxCacheSize, err = strconvBytes(maxCacheSizeStr)
|
||||
fatalIf(err, "Unable to convert MINIO_CACHE_SIZE=%s environment variable into its integer value.", maxCacheSizeStr)
|
||||
}
|
||||
|
||||
// Fetch cache expiry from environment variable.
|
||||
if cacheExpiryStr := os.Getenv("MINIO_CACHE_EXPIRY"); cacheExpiryStr != "" {
|
||||
// We need to parse cache expiry to its time.Duration value.
|
||||
globalCacheExpiry, err = time.ParseDuration(cacheExpiryStr)
|
||||
fatalIf(err, "Unable to convert MINIO_CACHE_EXPIRY=%s environment variable into its time.Duration value.", cacheExpiryStr)
|
||||
}
|
||||
|
||||
// When credentials inherited from the env, server cmd has to save them in the disk
|
||||
if os.Getenv("MINIO_ACCESS_KEY") != "" && os.Getenv("MINIO_SECRET_KEY") != "" {
|
||||
// Env credentials are already loaded in serverConfig, just save in the disk
|
||||
@@ -332,26 +297,6 @@ func checkServerSyntax(c *cli.Context) {
|
||||
fatalIf(err, "Storage endpoint error.")
|
||||
}
|
||||
|
||||
// Verify syntax for all the ignored disks.
|
||||
var ignoredEndpoints []*url.URL
|
||||
ignoredDisksStr := c.String("ignore-disks")
|
||||
if ignoredDisksStr != "" {
|
||||
ignoredDisks := strings.Split(ignoredDisksStr, ",")
|
||||
if len(endpoints) == 1 {
|
||||
fatalIf(errInvalidArgument, "--ignore-disks is valid only for XL setup.")
|
||||
}
|
||||
ignoredEndpoints, err = parseStorageEndpoints(ignoredDisks)
|
||||
fatalIf(err, "Unable to parse ignored storage endpoints %s", ignoredDisks)
|
||||
checkEndpointsSyntax(ignoredEndpoints, ignoredDisks)
|
||||
|
||||
for i, ep := range ignoredEndpoints {
|
||||
// An ignored disk should be present in the XL disks list.
|
||||
if !containsEndpoint(endpoints, ep) {
|
||||
fatalIf(errInvalidArgument, "Ignored storage %s not available in the list of erasure storages list.", disks[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !isDistributedSetup(endpoints) {
|
||||
// for FS and singlenode-XL validation is done, return.
|
||||
return
|
||||
@@ -412,18 +357,11 @@ func serverMain(c *cli.Context) {
|
||||
// depends on it.
|
||||
checkServerSyntax(c)
|
||||
|
||||
// Disks to be ignored in server init, to skip format healing.
|
||||
var ignoredEndpoints []*url.URL
|
||||
if len(c.String("ignore-disks")) > 0 {
|
||||
ignoredEndpoints, err = parseStorageEndpoints(strings.Split(c.String("ignore-disks"), ","))
|
||||
fatalIf(err, "Unable to parse storage endpoints %s", strings.Split(c.String("ignore-disks"), ","))
|
||||
}
|
||||
|
||||
// Disks to be used in server init.
|
||||
endpoints, err := parseStorageEndpoints(c.Args())
|
||||
fatalIf(err, "Unable to parse storage endpoints %s", c.Args())
|
||||
|
||||
storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints)
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
fatalIf(err, "Unable to initialize storage disks.")
|
||||
|
||||
// Cleanup objects that weren't successfully written into the namespace.
|
||||
@@ -440,10 +378,9 @@ func serverMain(c *cli.Context) {
|
||||
|
||||
// Configure server.
|
||||
srvConfig := serverCmdConfig{
|
||||
serverAddr: serverAddr,
|
||||
endpoints: endpoints,
|
||||
ignoredEndpoints: ignoredEndpoints,
|
||||
storageDisks: storageDisks,
|
||||
serverAddr: serverAddr,
|
||||
endpoints: endpoints,
|
||||
storageDisks: storageDisks,
|
||||
}
|
||||
|
||||
// Configure server.
|
||||
|
||||
@@ -258,7 +258,7 @@ func TestCheckServerSyntax(t *testing.T) {
|
||||
t.Fatalf("Test %d : Unexpected error %s", i+1, err)
|
||||
}
|
||||
checkEndpointsSyntax(endpoints, disks)
|
||||
_, err = initStorageDisks(endpoints, nil)
|
||||
_, err = initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Errorf("Test %d : disk init failed : %s", i+1, err)
|
||||
}
|
||||
@@ -336,9 +336,6 @@ func TestInitServerConfig(t *testing.T) {
|
||||
envVar string
|
||||
val string
|
||||
}{
|
||||
{"MINIO_MAXCONN", "10"},
|
||||
{"MINIO_CACHE_SIZE", "42MB"},
|
||||
{"MINIO_CACHE_EXPIRY", "2h45m"},
|
||||
{"MINIO_ACCESS_KEY", "abcd1"},
|
||||
{"MINIO_SECRET_KEY", "abcd12345"},
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -45,6 +44,10 @@ func getFormatStr(strLen int, padding int) string {
|
||||
|
||||
// Prints the formatted startup message.
|
||||
func printStartupMessage(endPoints []string) {
|
||||
// If quiet flag is set do not print startup message.
|
||||
if globalQuiet {
|
||||
return
|
||||
}
|
||||
printServerCommonMsg(endPoints)
|
||||
printCLIAccessMsg(endPoints[0])
|
||||
printObjectAPIMsg()
|
||||
@@ -73,11 +76,7 @@ func printServerCommonMsg(endPoints []string) {
|
||||
// Colorize the message and print.
|
||||
console.Println(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(endPointStr), 1), endPointStr)))
|
||||
console.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKeyID)))
|
||||
secretKey := cred.SecretAccessKey
|
||||
if strings.EqualFold(os.Getenv("MINIO_SECURE_CONSOLE"), "no") {
|
||||
secretKey = "*REDACTED*"
|
||||
}
|
||||
console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", secretKey)))
|
||||
console.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretAccessKey)))
|
||||
console.Println(colorBlue("Region: ") + colorBold(fmt.Sprintf(getFormatStr(len(region), 3), region)))
|
||||
printEventNotifiers()
|
||||
|
||||
@@ -109,15 +108,11 @@ func printCLIAccessMsg(endPoint string) {
|
||||
|
||||
// Configure 'mc', following block prints platform specific information for minio client.
|
||||
console.Println(colorBlue("\nCommand-line Access: ") + mcQuickStartGuide)
|
||||
secretKey := cred.SecretAccessKey
|
||||
if os.Getenv("MINIO_SECURE_CONSOLE") == "0" {
|
||||
secretKey = "*REDACTED*"
|
||||
}
|
||||
if runtime.GOOS == "windows" {
|
||||
mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKeyID, secretKey)
|
||||
mcMessage := fmt.Sprintf("$ mc.exe config host add myminio %s %s %s", endPoint, cred.AccessKeyID, cred.SecretAccessKey)
|
||||
console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
|
||||
} else {
|
||||
mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKeyID, secretKey)
|
||||
mcMessage := fmt.Sprintf("$ mc config host add myminio %s %s %s", endPoint, cred.AccessKeyID, cred.SecretAccessKey)
|
||||
console.Println(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -227,10 +227,6 @@ func (s *storageServer) TryInitHandler(args *GenericArgs, reply *GenericReply) e
|
||||
// Initialize new storage rpc.
|
||||
func newRPCServer(srvConfig serverCmdConfig) (servers []*storageServer, err error) {
|
||||
for _, ep := range srvConfig.endpoints {
|
||||
if containsEndpoint(srvConfig.ignoredEndpoints, ep) {
|
||||
// Do not init disk RPC for ignored end point.
|
||||
continue
|
||||
}
|
||||
// e.g server:/mnt/disk1
|
||||
if isLocalStorage(ep) {
|
||||
// Get the posix path.
|
||||
|
||||
@@ -66,7 +66,7 @@ func prepareFS() (ObjectLayer, string, error) {
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
removeRoots(fsDirs)
|
||||
return nil, "", err
|
||||
@@ -84,7 +84,7 @@ func prepareXL() (ObjectLayer, []string, error) {
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
obj, _, err := initObjectLayer(endpoints, nil)
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
removeRoots(fsDirs)
|
||||
return nil, nil, err
|
||||
@@ -189,7 +189,7 @@ func UnstartedTestServer(t TestErrHandler, instanceType string) TestServer {
|
||||
testServer.AccessKey = credentials.AccessKeyID
|
||||
testServer.SecretKey = credentials.SecretAccessKey
|
||||
|
||||
objLayer, storageDisks, err := initObjectLayer(testServer.Disks, nil)
|
||||
objLayer, storageDisks, err := initObjectLayer(testServer.Disks)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
|
||||
}
|
||||
@@ -385,7 +385,7 @@ func StartTestPeersRPCServer(t TestErrHandler, instanceType string) TestServer {
|
||||
testRPCServer.SecretKey = credentials.SecretAccessKey
|
||||
|
||||
// create temporary backend for the test server.
|
||||
objLayer, storageDisks, err := initObjectLayer(endpoints, nil)
|
||||
objLayer, storageDisks, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
|
||||
}
|
||||
@@ -459,7 +459,7 @@ func StartTestControlRPCServer(t TestErrHandler, instanceType string) TestServer
|
||||
testRPCServer.SecretKey = credentials.SecretAccessKey
|
||||
|
||||
// create temporary backend for the test server.
|
||||
objLayer, storageDisks, err := initObjectLayer(endpoints, nil)
|
||||
objLayer, storageDisks, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed obtaining Temp Backend: <ERROR> %s", err)
|
||||
}
|
||||
@@ -1595,8 +1595,8 @@ func getRandomDisks(N int) ([]string, error) {
|
||||
}
|
||||
|
||||
// initObjectLayer - Instantiates object layer and returns it.
|
||||
func initObjectLayer(endpoints, ignoredEndpoints []*url.URL) (ObjectLayer, []StorageAPI, error) {
|
||||
storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints)
|
||||
func initObjectLayer(endpoints []*url.URL) (ObjectLayer, []StorageAPI, error) {
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
@@ -1677,7 +1677,7 @@ func prepareXLStorageDisks(t *testing.T) ([]StorageAPI, []string) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
_, storageDisks, err := initObjectLayer(endpoints, nil)
|
||||
_, storageDisks, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
removeRoots(fsDirs)
|
||||
t.Fatal("Unable to initialize storage disks", err)
|
||||
@@ -1972,7 +1972,7 @@ func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType)
|
||||
if err != nil {
|
||||
t.Fatalf("Initialization of disks for XL setup: %s", err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(endpoints, nil)
|
||||
objLayer, _, err := initObjectLayer(endpoints)
|
||||
if err != nil {
|
||||
t.Fatalf("Initialization of object layer failed for XL setup: %s", err)
|
||||
}
|
||||
|
||||
+4
-7
@@ -23,7 +23,6 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"encoding/json"
|
||||
@@ -171,16 +170,14 @@ func urlPathSplit(urlPath string) (bucketName, prefixName string) {
|
||||
func startProfiler(profiler string) interface {
|
||||
Stop()
|
||||
} {
|
||||
// Set ``MINIO_PROFILE_DIR`` to the directory where profiling information should be persisted
|
||||
profileDir := os.Getenv("MINIO_PROFILE_DIR")
|
||||
// Enable profiler if ``MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
|
||||
// Enable profiler if ``_MINIO_PROFILER`` is set. Supported options are [cpu, mem, block].
|
||||
switch profiler {
|
||||
case "cpu":
|
||||
return profile.Start(profile.CPUProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
|
||||
return profile.Start(profile.CPUProfile, profile.NoShutdownHook)
|
||||
case "mem":
|
||||
return profile.Start(profile.MemProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
|
||||
return profile.Start(profile.MemProfile, profile.NoShutdownHook)
|
||||
case "block":
|
||||
return profile.Start(profile.BlockProfile, profile.NoShutdownHook, profile.ProfilePath(profileDir))
|
||||
return profile.Start(profile.BlockProfile, profile.NoShutdownHook)
|
||||
default:
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -220,6 +220,13 @@ func TestMaxPartID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Add tests for starting and stopping different profilers.
|
||||
func TestStartProfiler(t *testing.T) {
|
||||
if startProfiler("") != nil {
|
||||
t.Fatal("Expected nil, but non-nil value returned for invalid profiler.")
|
||||
}
|
||||
}
|
||||
|
||||
// Tests fetch local address.
|
||||
func TestLocalAddress(t *testing.T) {
|
||||
if runtime.GOOS == "windows" {
|
||||
|
||||
+6
-1
@@ -18,7 +18,9 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/objcache"
|
||||
@@ -129,6 +131,9 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
|
||||
// Initialize list pool.
|
||||
listPool := newTreeWalkPool(globalLookupTimeout)
|
||||
|
||||
// Check if object cache is disabled.
|
||||
objCacheDisabled := strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
|
||||
|
||||
// Initialize xl objects.
|
||||
xl := xlObjects{
|
||||
storageDisks: newStorageDisks,
|
||||
@@ -136,7 +141,7 @@ func newXLObjects(storageDisks []StorageAPI) (ObjectLayer, error) {
|
||||
parityBlocks: parityBlocks,
|
||||
listPool: listPool,
|
||||
objCache: objCache,
|
||||
objCacheEnabled: globalMaxCacheSize > 0,
|
||||
objCacheEnabled: !objCacheDisabled,
|
||||
}
|
||||
|
||||
// Figure out read and write quorum based on number of storage disks.
|
||||
|
||||
+3
-12
@@ -56,12 +56,7 @@ func TestStorageInfo(t *testing.T) {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
|
||||
ignoredEndpoints, err := parseStorageEndpoints(fsDirs[:4])
|
||||
if err != nil {
|
||||
t.Fatalf("Unexpected error %s", err)
|
||||
}
|
||||
|
||||
storageDisks, err := initStorageDisks(endpoints, ignoredEndpoints)
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
@@ -156,7 +151,7 @@ func TestNewXL(t *testing.T) {
|
||||
t.Fatalf("Unable to initialize erasure, %s", err)
|
||||
}
|
||||
|
||||
storageDisks, err := initStorageDisks(endpoints, nil)
|
||||
storageDisks, err := initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
@@ -186,11 +181,7 @@ func TestNewXL(t *testing.T) {
|
||||
t.Fatalf("Unable to initialize erasure, %s", err)
|
||||
}
|
||||
|
||||
ignoredEndpoints, err := parseStorageEndpoints(erasureDisks[:2])
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to initialize erasure, %s", err)
|
||||
}
|
||||
storageDisks, err = initStorageDisks(endpoints, ignoredEndpoints)
|
||||
storageDisks, err = initStorageDisks(endpoints)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user