mirror of
https://github.com/pgsty/minio.git
synced 2026-11-16 20:39:35 +03:00
Compare commits
33 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e5e522fc61 | |||
| de251483d1 | |||
| 805186ab97 | |||
| ea76e72054 | |||
| 8bd7a19d50 | |||
| 25de775560 | |||
| abf209b1dd | |||
| d9d13c898c | |||
| ad79c626c6 | |||
| cd152f404a | |||
| 5fbdd70de9 | |||
| 0bbdd02a57 | |||
| 78abe5234e | |||
| f46ee54194 | |||
| 6005dbf01f | |||
| eb0e56ccf6 | |||
| c91abe6c4b | |||
| 670b538dde | |||
| 186000328e | |||
| 2575f4198a | |||
| 05a64dee95 | |||
| 577d10674d | |||
| 81ee79b042 | |||
| d94500ae26 | |||
| 001d9a4ae7 | |||
| c3a5146422 | |||
| 36c39d04da | |||
| 28d526bc68 | |||
| 05f96f3956 | |||
| cb9ee1584a | |||
| 9f4c120731 | |||
| 12a916091e | |||
| 842092f8de |
@@ -50,9 +50,11 @@ deadcode:
|
||||
@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
|
||||
|
||||
spelling:
|
||||
@${GOPATH}/bin/misspell -error `find cmd/`
|
||||
@${GOPATH}/bin/misspell -error `find pkg/`
|
||||
@${GOPATH}/bin/misspell -error `find docs/`
|
||||
@${GOPATH}/bin/misspell -locale US -error `find cmd/`
|
||||
@${GOPATH}/bin/misspell -locale US -error `find pkg/`
|
||||
@${GOPATH}/bin/misspell -locale US -error `find docs/`
|
||||
@${GOPATH}/bin/misspell -locale US -error `find buildscripts/`
|
||||
@${GOPATH}/bin/misspell -locale US -error `find dockerscripts/`
|
||||
|
||||
# Builds minio, runs the verifiers then runs the tests.
|
||||
check: test
|
||||
@@ -89,7 +91,7 @@ pkg-list:
|
||||
# Builds minio and installs it to $GOPATH/bin.
|
||||
install: build
|
||||
@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
|
||||
@cp $(PWD)/minio $(GOPATH)/bin/minio
|
||||
@mkdir -p $(GOPATH)/bin && cp $(PWD)/minio $(GOPATH)/bin/minio
|
||||
@echo "Installation successful. To learn more, try \"minio --help\"."
|
||||
|
||||
clean:
|
||||
|
||||
@@ -25,14 +25,35 @@ import web from "../web"
|
||||
import { Redirect } from "react-router-dom"
|
||||
|
||||
export class Login extends React.Component {
|
||||
constructor(props) {
|
||||
super(props)
|
||||
this.state = {
|
||||
accessKey: "",
|
||||
secretKey: ""
|
||||
}
|
||||
}
|
||||
|
||||
// Handle field changes
|
||||
accessKeyChange(e) {
|
||||
this.setState({
|
||||
accessKey: e.target.value
|
||||
})
|
||||
}
|
||||
|
||||
secretKeyChange(e) {
|
||||
this.setState({
|
||||
secretKey: e.target.value
|
||||
})
|
||||
}
|
||||
|
||||
handleSubmit(event) {
|
||||
event.preventDefault()
|
||||
const { showAlert, history } = this.props
|
||||
let message = ""
|
||||
if (!document.getElementById("accessKey").value) {
|
||||
if (this.state.accessKey === "") {
|
||||
message = "Access Key cannot be empty"
|
||||
}
|
||||
if (!document.getElementById("secretKey").value) {
|
||||
if (this.state.secretKey === "") {
|
||||
message = "Secret Key cannot be empty"
|
||||
}
|
||||
if (message) {
|
||||
@@ -41,8 +62,8 @@ export class Login extends React.Component {
|
||||
}
|
||||
web
|
||||
.Login({
|
||||
username: document.getElementById("accessKey").value,
|
||||
password: document.getElementById("secretKey").value
|
||||
username: this.state.accessKey,
|
||||
password: this.state.secretKey
|
||||
})
|
||||
.then(res => {
|
||||
history.push("/")
|
||||
@@ -77,6 +98,8 @@ export class Login extends React.Component {
|
||||
<div className="l-wrap">
|
||||
<form onSubmit={this.handleSubmit.bind(this)}>
|
||||
<InputGroup
|
||||
value={this.state.accessKey}
|
||||
onChange={this.accessKeyChange.bind(this)}
|
||||
className="ig-dark"
|
||||
label="Access Key"
|
||||
id="accessKey"
|
||||
@@ -87,6 +110,8 @@ export class Login extends React.Component {
|
||||
autoComplete="username"
|
||||
/>
|
||||
<InputGroup
|
||||
value={this.state.secretKey}
|
||||
onChange={this.secretKeyChange.bind(this)}
|
||||
className="ig-dark"
|
||||
label="Secret Key"
|
||||
id="secretKey"
|
||||
|
||||
@@ -60,20 +60,25 @@ describe("Login", () => {
|
||||
alert={{ show: false, type: "danger"}}
|
||||
showAlert={showAlertMock}
|
||||
clearAlert={clearAlertMock}
|
||||
/>,
|
||||
{ attachTo: document.body }
|
||||
/>
|
||||
)
|
||||
// case where both keys are empty - displays the second warning
|
||||
wrapper.find("form").simulate("submit")
|
||||
expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
|
||||
|
||||
// case where access key is empty
|
||||
document.getElementById("secretKey").value = "secretKey"
|
||||
wrapper.setState({
|
||||
accessKey: "",
|
||||
secretKey: "secretKey"
|
||||
})
|
||||
wrapper.find("form").simulate("submit")
|
||||
expect(showAlertMock).toHaveBeenCalledWith("danger", "Access Key cannot be empty")
|
||||
|
||||
// case where secret key is empty
|
||||
document.getElementById("accessKey").value = "accessKey"
|
||||
wrapper.setState({
|
||||
accessKey: "accessKey",
|
||||
secretKey: ""
|
||||
})
|
||||
wrapper.find("form").simulate("submit")
|
||||
expect(showAlertMock).toHaveBeenCalledWith("danger", "Secret Key cannot be empty")
|
||||
})
|
||||
@@ -85,11 +90,12 @@ describe("Login", () => {
|
||||
alert={{ show: false, type: "danger"}}
|
||||
showAlert={showAlertMock}
|
||||
clearAlert={clearAlertMock}
|
||||
/>,
|
||||
{ attachTo: document.body }
|
||||
/>
|
||||
)
|
||||
document.getElementById("accessKey").value = "accessKey"
|
||||
document.getElementById("secretKey").value = "secretKey"
|
||||
wrapper.setState({
|
||||
accessKey: "accessKey",
|
||||
secretKey: "secretKey"
|
||||
})
|
||||
wrapper.find("form").simulate("submit")
|
||||
expect(web.Login).toHaveBeenCalledWith({
|
||||
"username": "accessKey",
|
||||
|
||||
+23
-23
File diff suppressed because one or more lines are too long
@@ -34,7 +34,7 @@ _init() {
|
||||
|
||||
## FIXME:
|
||||
## In OSX, 'readlink -f' option does not exist, hence
|
||||
## we have our own readlink -f behaviour here.
|
||||
## we have our own readlink -f behavior here.
|
||||
## Once OSX has the option, below function is good enough.
|
||||
##
|
||||
## readlink() {
|
||||
|
||||
@@ -33,6 +33,7 @@ func genLDFlags(version string) string {
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH")
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT")
|
||||
return ldflagsStr
|
||||
}
|
||||
|
||||
|
||||
@@ -57,7 +57,7 @@ const (
|
||||
var (
|
||||
errHealIdleTimeout = fmt.Errorf("healing results were not consumed for too long")
|
||||
errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
|
||||
errHealStopSignalled = fmt.Errorf("heal stop signalled")
|
||||
errHealStopSignalled = fmt.Errorf("heal stop signaled")
|
||||
|
||||
errFnHealFromAPIErr = func(err error) error {
|
||||
errCode := toAPIErrorCode(err)
|
||||
@@ -301,7 +301,7 @@ type healSequence struct {
|
||||
// current accumulated status of the heal sequence
|
||||
currentStatus healSequenceStatus
|
||||
|
||||
// channel signalled by background routine when traversal has
|
||||
// channel signaled by background routine when traversal has
|
||||
// completed
|
||||
traverseAndHealDoneCh chan error
|
||||
|
||||
@@ -441,7 +441,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
|
||||
h.currentStatus.updateLock.Unlock()
|
||||
|
||||
// This is a "safe" point for the heal sequence to quit if
|
||||
// signalled externally.
|
||||
// signaled externally.
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
|
||||
@@ -179,9 +179,9 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
|
||||
for _, hostStr := range GetRemotePeers(endpoints) {
|
||||
host, err := xnet.ParseHost(hostStr)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to parse Admin RPC Host", context.Background())
|
||||
rpcClient, err := NewAdminRPCClient(host)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to initialize Admin RPC Client", context.Background())
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
addr: hostStr,
|
||||
cmdRunner: rpcClient,
|
||||
|
||||
@@ -121,7 +121,7 @@ func NewAdminRPCServer() (*xrpc.Server, error) {
|
||||
// registerAdminRPCRouter - creates and registers Admin RPC server and its router.
|
||||
func registerAdminRPCRouter(router *mux.Router) {
|
||||
rpcServer, err := NewAdminRPCServer()
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
|
||||
subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
|
||||
subrouter.Path(adminServiceSubPath).Handler(rpcServer)
|
||||
}
|
||||
|
||||
+6
-1
@@ -193,6 +193,7 @@ const (
|
||||
ErrHealMissingBucket
|
||||
ErrHealAlreadyRunning
|
||||
ErrHealOverlappingPaths
|
||||
ErrIncorrectContinuationToken
|
||||
)
|
||||
|
||||
// error code to APIError structure, these fields carry respective
|
||||
@@ -842,7 +843,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Object storage backend is unreachable",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
|
||||
ErrIncorrectContinuationToken: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "The continuation token provided is incorrect",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// Add your error structure here.
|
||||
}
|
||||
|
||||
|
||||
+11
-1
@@ -36,7 +36,17 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
|
||||
}
|
||||
|
||||
// Parse bucket url queries for ListObjects V2.
|
||||
func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string) {
|
||||
func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) {
|
||||
errCode = ErrNone
|
||||
|
||||
// The continuation-token cannot be empty.
|
||||
if val, ok := values["continuation-token"]; ok {
|
||||
if len(val[0]) == 0 {
|
||||
errCode = ErrIncorrectContinuationToken
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
prefix = values.Get("prefix")
|
||||
token = values.Get("continuation-token")
|
||||
startAfter = values.Get("start-after")
|
||||
|
||||
@@ -29,6 +29,7 @@ func TestListObjectsV2Resources(t *testing.T) {
|
||||
fetchOwner bool
|
||||
maxKeys int
|
||||
encodingType string
|
||||
errCode APIErrorCode
|
||||
}{
|
||||
{
|
||||
values: url.Values{
|
||||
@@ -47,6 +48,7 @@ func TestListObjectsV2Resources(t *testing.T) {
|
||||
fetchOwner: true,
|
||||
maxKeys: 100,
|
||||
encodingType: "gzip",
|
||||
errCode: ErrNone,
|
||||
},
|
||||
{
|
||||
values: url.Values{
|
||||
@@ -64,11 +66,34 @@ func TestListObjectsV2Resources(t *testing.T) {
|
||||
fetchOwner: true,
|
||||
maxKeys: 1000,
|
||||
encodingType: "gzip",
|
||||
errCode: ErrNone,
|
||||
},
|
||||
{
|
||||
values: url.Values{
|
||||
"prefix": []string{"photos/"},
|
||||
"continuation-token": []string{""},
|
||||
"start-after": []string{"start-after"},
|
||||
"delimiter": []string{"/"},
|
||||
"fetch-owner": []string{"true"},
|
||||
"encoding-type": []string{"gzip"},
|
||||
},
|
||||
prefix: "",
|
||||
token: "",
|
||||
startAfter: "",
|
||||
delimiter: "",
|
||||
fetchOwner: false,
|
||||
maxKeys: 0,
|
||||
encodingType: "",
|
||||
errCode: ErrIncorrectContinuationToken,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType := getListObjectsV2Args(testCase.values)
|
||||
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values)
|
||||
|
||||
if errCode != testCase.errCode {
|
||||
t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode)
|
||||
}
|
||||
if prefix != testCase.prefix {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
|
||||
}
|
||||
|
||||
@@ -70,8 +70,15 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
urlValues := r.URL.Query()
|
||||
|
||||
// Extract all the listObjectsV2 query params to their native values.
|
||||
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _ := getListObjectsV2Args(r.URL.Query())
|
||||
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
|
||||
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponse(w, errCode, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// In ListObjectsV2 'continuation-token' is the marker.
|
||||
marker := token
|
||||
@@ -93,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
}
|
||||
// Inititate a list objects operation based on the input params.
|
||||
// On success would return back ListObjectsInfo object to be
|
||||
// marshalled into S3 compatible XML header.
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, fetchOwner, startAfter)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
@@ -159,7 +166,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
}
|
||||
// Inititate a list objects operation based on the input params.
|
||||
// On success would return back ListObjectsInfo object to be
|
||||
// marshalled into S3 compatible XML header.
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
|
||||
@@ -282,7 +282,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
// Allocate incoming content length bytes.
|
||||
deleteXMLBytes := make([]byte, r.ContentLength)
|
||||
var deleteXMLBytes []byte
|
||||
const maxBodySize = 2 * 1000 * 1024 // The max. XML contains 1000 object names (each at most 1024 bytes long) + XML overhead
|
||||
if r.ContentLength > maxBodySize { // Only allocated memory for at most 1000 objects
|
||||
deleteXMLBytes = make([]byte, maxBodySize)
|
||||
} else {
|
||||
deleteXMLBytes = make([]byte, r.ContentLength)
|
||||
}
|
||||
|
||||
// Read incoming body XML bytes.
|
||||
if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
|
||||
|
||||
@@ -224,11 +224,17 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
return
|
||||
}
|
||||
|
||||
host, e := xnet.ParseHost(r.RemoteAddr)
|
||||
logger.CriticalIf(ctx, e)
|
||||
host, err := xnet.ParseHost(r.RemoteAddr)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
target, e := target.NewHTTPClientTarget(*host, w)
|
||||
logger.CriticalIf(ctx, e)
|
||||
target, err := target.NewHTTPClientTarget(*host, w)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
rulesMap := event.NewRulesMap(eventNames, pattern, target.ID())
|
||||
|
||||
@@ -241,10 +247,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
|
||||
defer globalNotificationSys.RemoveRulesMap(bucketName, rulesMap)
|
||||
|
||||
thisAddr, e := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
logger.CriticalIf(ctx, e)
|
||||
thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
|
||||
if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
@@ -259,7 +268,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
<-target.DoneCh
|
||||
|
||||
if err := RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
|
||||
if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
|
||||
@@ -22,6 +22,9 @@ var (
|
||||
// GOPATH - GOPATH value at the time of build.
|
||||
GOPATH = ""
|
||||
|
||||
// GOROOT - GOROOT value at the time of build.
|
||||
GOROOT = ""
|
||||
|
||||
// Go get development tag.
|
||||
goGetTag = "DEVELOPMENT.GOGET"
|
||||
|
||||
|
||||
+5
-4
@@ -86,10 +86,11 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
rootCAs, err := x509.SystemCertPool()
|
||||
if err != nil {
|
||||
// In some systems like Windows, system cert pool is not supported.
|
||||
// Hence we create a new cert pool.
|
||||
rootCAs, _ := x509.SystemCertPool()
|
||||
if rootCAs == nil {
|
||||
// In some systems (like Windows) system cert pool is
|
||||
// not supported or no certificates are present on the
|
||||
// system - so we create a new cert pool.
|
||||
rootCAs = x509.NewCertPool()
|
||||
}
|
||||
|
||||
|
||||
@@ -205,6 +205,16 @@ func handleCommonEnvVars() {
|
||||
globalCacheExpiry = expiry
|
||||
}
|
||||
|
||||
if maxUseStr := os.Getenv("MINIO_CACHE_MAXUSE"); maxUseStr != "" {
|
||||
maxUse, err := strconv.Atoi(maxUseStr)
|
||||
if err != nil {
|
||||
logger.Fatal(uiErrInvalidCacheMaxUse(err), "Unable to parse MINIO_CACHE_MAXUSE value (`%s`)", maxUseStr)
|
||||
}
|
||||
// maxUse should be a valid percentage.
|
||||
if maxUse > 0 && maxUse <= 100 {
|
||||
globalCacheMaxUse = maxUse
|
||||
}
|
||||
}
|
||||
// In place update is true by default if the MINIO_UPDATE is not set
|
||||
// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
|
||||
// in-place update is off.
|
||||
|
||||
@@ -39,9 +39,9 @@ import (
|
||||
// 6. Make changes in config-current_test.go for any test change
|
||||
|
||||
// Config version
|
||||
const serverConfigVersion = "25"
|
||||
const serverConfigVersion = "26"
|
||||
|
||||
type serverConfig = serverConfigV25
|
||||
type serverConfig = serverConfigV26
|
||||
|
||||
var (
|
||||
// globalServerConfig server config.
|
||||
@@ -116,10 +116,11 @@ func (s *serverConfig) GetWorm() bool {
|
||||
}
|
||||
|
||||
// SetCacheConfig sets the current cache config
|
||||
func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int) {
|
||||
func (s *serverConfig) SetCacheConfig(drives, exclude []string, expiry int, maxuse int) {
|
||||
s.Cache.Drives = drives
|
||||
s.Cache.Exclude = exclude
|
||||
s.Cache.Expiry = expiry
|
||||
s.Cache.MaxUse = maxuse
|
||||
}
|
||||
|
||||
// GetCacheConfig gets the current cache config
|
||||
@@ -205,6 +206,7 @@ func newServerConfig() *serverConfig {
|
||||
Drives: []string{},
|
||||
Exclude: []string{},
|
||||
Expiry: globalCacheExpiry,
|
||||
MaxUse: globalCacheMaxUse,
|
||||
},
|
||||
Notify: notifier{},
|
||||
}
|
||||
@@ -232,6 +234,7 @@ func newServerConfig() *serverConfig {
|
||||
srvCfg.Cache.Drives = make([]string, 0)
|
||||
srvCfg.Cache.Exclude = make([]string, 0)
|
||||
srvCfg.Cache.Expiry = globalCacheExpiry
|
||||
srvCfg.Cache.MaxUse = globalCacheMaxUse
|
||||
return srvCfg
|
||||
}
|
||||
|
||||
@@ -270,7 +273,7 @@ func newConfig() error {
|
||||
}
|
||||
|
||||
if globalIsDiskCacheEnabled {
|
||||
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
|
||||
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
@@ -358,7 +361,7 @@ func loadConfig() error {
|
||||
}
|
||||
|
||||
if globalIsDiskCacheEnabled {
|
||||
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry)
|
||||
srvCfg.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
@@ -387,6 +390,7 @@ func loadConfig() error {
|
||||
globalCacheDrives = cacheConf.Drives
|
||||
globalCacheExcludes = cacheConf.Exclude
|
||||
globalCacheExpiry = cacheConf.Expiry
|
||||
globalCacheMaxUse = cacheConf.MaxUse
|
||||
}
|
||||
globalServerConfigMu.Unlock()
|
||||
|
||||
|
||||
@@ -183,6 +183,11 @@ func migrateConfig() error {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case "25":
|
||||
if err = migrateV25ToV26(); err != nil {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case serverConfigVersion:
|
||||
// No migration needed. this always points to current version.
|
||||
err = nil
|
||||
@@ -2193,3 +2198,122 @@ func migrateV24ToV25() error {
|
||||
logger.Info(configMigrateMSGTemplate, configFile, cv24.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV25ToV26() error {
|
||||
configFile := getConfigFile()
|
||||
|
||||
cv25 := &serverConfigV25{}
|
||||
_, err := quick.LoadConfig(configFile, globalEtcdClient, cv25)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config version ‘25’. %v", err)
|
||||
}
|
||||
if cv25.Version != "25" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy over fields from V25 into V26 config struct
|
||||
srvConfig := &serverConfigV26{
|
||||
Notify: notifier{},
|
||||
}
|
||||
srvConfig.Version = "26"
|
||||
srvConfig.Credential = cv25.Credential
|
||||
srvConfig.Region = cv25.Region
|
||||
if srvConfig.Region == "" {
|
||||
// Region needs to be set for AWS Signature Version 4.
|
||||
srvConfig.Region = globalMinioDefaultRegion
|
||||
}
|
||||
|
||||
if len(cv25.Notify.AMQP) == 0 {
|
||||
srvConfig.Notify.AMQP = make(map[string]target.AMQPArgs)
|
||||
srvConfig.Notify.AMQP["1"] = target.AMQPArgs{}
|
||||
} else {
|
||||
srvConfig.Notify.AMQP = cv25.Notify.AMQP
|
||||
}
|
||||
if len(cv25.Notify.Elasticsearch) == 0 {
|
||||
srvConfig.Notify.Elasticsearch = make(map[string]target.ElasticsearchArgs)
|
||||
srvConfig.Notify.Elasticsearch["1"] = target.ElasticsearchArgs{
|
||||
Format: event.NamespaceFormat,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.Elasticsearch = cv25.Notify.Elasticsearch
|
||||
}
|
||||
if len(cv25.Notify.Redis) == 0 {
|
||||
srvConfig.Notify.Redis = make(map[string]target.RedisArgs)
|
||||
srvConfig.Notify.Redis["1"] = target.RedisArgs{
|
||||
Format: event.NamespaceFormat,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.Redis = cv25.Notify.Redis
|
||||
}
|
||||
if len(cv25.Notify.PostgreSQL) == 0 {
|
||||
srvConfig.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
|
||||
srvConfig.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{
|
||||
Format: event.NamespaceFormat,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.PostgreSQL = cv25.Notify.PostgreSQL
|
||||
}
|
||||
if len(cv25.Notify.Kafka) == 0 {
|
||||
srvConfig.Notify.Kafka = make(map[string]target.KafkaArgs)
|
||||
srvConfig.Notify.Kafka["1"] = target.KafkaArgs{}
|
||||
} else {
|
||||
srvConfig.Notify.Kafka = cv25.Notify.Kafka
|
||||
}
|
||||
if len(cv25.Notify.NATS) == 0 {
|
||||
srvConfig.Notify.NATS = make(map[string]target.NATSArgs)
|
||||
srvConfig.Notify.NATS["1"] = target.NATSArgs{}
|
||||
} else {
|
||||
srvConfig.Notify.NATS = cv25.Notify.NATS
|
||||
}
|
||||
if len(cv25.Notify.Webhook) == 0 {
|
||||
srvConfig.Notify.Webhook = make(map[string]target.WebhookArgs)
|
||||
srvConfig.Notify.Webhook["1"] = target.WebhookArgs{}
|
||||
} else {
|
||||
srvConfig.Notify.Webhook = cv25.Notify.Webhook
|
||||
}
|
||||
if len(cv25.Notify.MySQL) == 0 {
|
||||
srvConfig.Notify.MySQL = make(map[string]target.MySQLArgs)
|
||||
srvConfig.Notify.MySQL["1"] = target.MySQLArgs{
|
||||
Format: event.NamespaceFormat,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.MySQL = cv25.Notify.MySQL
|
||||
}
|
||||
|
||||
if len(cv25.Notify.MQTT) == 0 {
|
||||
srvConfig.Notify.MQTT = make(map[string]target.MQTTArgs)
|
||||
srvConfig.Notify.MQTT["1"] = target.MQTTArgs{}
|
||||
} else {
|
||||
srvConfig.Notify.MQTT = cv25.Notify.MQTT
|
||||
}
|
||||
|
||||
// Load browser config from existing config in the file.
|
||||
srvConfig.Browser = cv25.Browser
|
||||
|
||||
// Load worm config from existing config in the file.
|
||||
srvConfig.Worm = cv25.Worm
|
||||
|
||||
// Load domain config from existing config in the file.
|
||||
srvConfig.Domain = cv25.Domain
|
||||
|
||||
// Load storage class config from existing storage class config in the file.
|
||||
srvConfig.StorageClass.RRS = cv25.StorageClass.RRS
|
||||
srvConfig.StorageClass.Standard = cv25.StorageClass.Standard
|
||||
|
||||
// Load cache config from existing cache config in the file.
|
||||
srvConfig.Cache.Drives = cv25.Cache.Drives
|
||||
srvConfig.Cache.Exclude = cv25.Cache.Exclude
|
||||
srvConfig.Cache.Expiry = cv25.Cache.Expiry
|
||||
|
||||
// Add predefined value to new server config.
|
||||
srvConfig.Cache.MaxUse = globalCacheMaxUse
|
||||
|
||||
if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv25.Version, srvConfig.Version, err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, cv25.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -654,3 +654,30 @@ type serverConfigV25 struct {
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV26 is just like version '25', stores additionally
|
||||
// cache max use value in 'CacheConfig'.
|
||||
//
|
||||
// IMPORTANT NOTE: When updating this struct make sure that
|
||||
// serverConfig.ConfigDiff() is updated as necessary.
|
||||
type serverConfigV26 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BoolFlag `json:"browser"`
|
||||
Worm BoolFlag `json:"worm"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
}
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Package crypto implements AWS S3 related cryptographic building blocks
|
||||
// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption
|
||||
// with customer provided keys (SSE-C).
|
||||
//
|
||||
// All objects are encrypted with an unique and randomly generated 'ObjectKey'.
|
||||
// The ObjectKey itself is never stored in plaintext. Instead it is only stored
|
||||
// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey'
|
||||
// with an unique key-encryption-key. Given the correct key-encryption-key the
|
||||
// sealed 'ObjectKey' can be unsealed and the object can be decrypted.
|
||||
//
|
||||
//
|
||||
// ## SSE-C
|
||||
//
|
||||
// SSE-C computes the key-encryption-key from the client-provided key, an
|
||||
// initialization vector (IV) and the bucket/object path.
|
||||
//
|
||||
// 1. Encrypt:
|
||||
// Input: ClientKey, bucket, object, metadata, object_data
|
||||
// - IV := Random({0,1}²⁵⁶)
|
||||
// - ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶))
|
||||
// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
|
||||
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
|
||||
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
|
||||
// - metadata <- IV
|
||||
// - metadata <- SealedKey
|
||||
// Output: enc_object_data, metadata
|
||||
//
|
||||
// 2. Decrypt:
|
||||
// Input: ClientKey, bucket, object, metadata, enc_object_data
|
||||
// - IV <- metadata
|
||||
// - SealedKey <- metadata
|
||||
// - KeyEncKey := HMAC-SHA256(ClientKey, IV || bucket || object)
|
||||
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
|
||||
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
|
||||
// Output: object_data
|
||||
//
|
||||
//
|
||||
// ## SSE-S3
|
||||
//
|
||||
// SSE-S3 can use either a master key or a KMS as root-of-trust.
|
||||
// The en/decryption slightly depens upon which root-of-trust is used.
|
||||
//
|
||||
// ### SSE-S3 and single master key
|
||||
//
|
||||
// The master key is used to derive unique object- and key-encryption-keys.
|
||||
// SSE-S3 with a single master key works as SSE-C where the master key is
|
||||
// used as the client-provided key.
|
||||
//
|
||||
// 1. Encrypt:
|
||||
// Input: MasterKey, bucket, object, metadata, object_data
|
||||
// - IV := Random({0,1}²⁵⁶)
|
||||
// - ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶))
|
||||
// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
|
||||
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
|
||||
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
|
||||
// - metadata <- IV
|
||||
// - metadata <- SealedKey
|
||||
// Output: enc_object_data, metadata
|
||||
//
|
||||
// 2. Decrypt:
|
||||
// Input: MasterKey, bucket, object, metadata, enc_object_data
|
||||
// - IV <- metadata
|
||||
// - SealedKey <- metadata
|
||||
// - KeyEncKey := HMAC-SHA256(MasterKey, IV || bucket || object)
|
||||
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
|
||||
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
|
||||
// Output: object_data
|
||||
//
|
||||
//
|
||||
// ### SSE-S3 and KMS
|
||||
//
|
||||
// SSE-S3 requires that the KMS provides two functions:
|
||||
// 1. Generate(KeyID) -> (Key, EncKey)
|
||||
// 2. Unseal(KeyID, EncKey) -> Key
|
||||
//
|
||||
// 1. Encrypt:
|
||||
// Input: KeyID, bucket, object, metadata, object_data
|
||||
// - Key, EncKey := Generate(KeyID)
|
||||
// - IV := Random({0,1}²⁵⁶)
|
||||
// - ObjectKey := SHA256(Key, Random({0,1}²⁵⁶))
|
||||
// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
|
||||
// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey)
|
||||
// - enc_object_data := DAREv2_Enc(ObjectKey, object_data)
|
||||
// - metadata <- IV
|
||||
// - metadata <- KeyID
|
||||
// - metadata <- EncKey
|
||||
// - metadata <- SealedKey
|
||||
// Output: enc_object_data, metadata
|
||||
//
|
||||
// 2. Decrypt:
|
||||
// Input: bucket, object, metadata, enc_object_data
|
||||
// - KeyID <- metadata
|
||||
// - EncKey <- metadata
|
||||
// - IV <- metadata
|
||||
// - SealedKey <- metadata
|
||||
// - Key := Unseal(KeyID, EncKey)
|
||||
// - KeyEncKey := HMAC-SHA256(Key, IV || bucket || object)
|
||||
// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey)
|
||||
// - object_data := DAREv2_Dec(ObjectKey, enc_object_data)
|
||||
// Output: object_data
|
||||
//
|
||||
package crypto
|
||||
@@ -0,0 +1,23 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
// ErrInvalidEncryptionMethod indicates that the specified SSE encryption method
|
||||
// is not supported.
|
||||
ErrInvalidEncryptionMethod = errors.New("The encryption method is not supported")
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// SSEHeader is the general AWS SSE HTTP header key.
|
||||
const SSEHeader = "X-Amz-Server-Side-Encryption"
|
||||
|
||||
// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
|
||||
// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
|
||||
// For SSE-C see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
|
||||
const SSEAlgorithmAES256 = "AES256"
|
||||
|
||||
// S3 represents AWS SSE-S3. It provides functionality to handle
|
||||
// SSE-S3 requests.
|
||||
var S3 = s3{}
|
||||
|
||||
type s3 struct{}
|
||||
|
||||
// IsRequested returns true if the HTTP headers indicates that
|
||||
// the S3 client requests SSE-S3.
|
||||
func (s3) IsRequested(h http.Header) bool {
|
||||
_, ok := h[SSEHeader]
|
||||
return ok
|
||||
}
|
||||
|
||||
// Parse parses the SSE-S3 related HTTP headers and checks
|
||||
// whether they contain valid values.
|
||||
func (s3) Parse(h http.Header) (err error) {
|
||||
if h.Get(SSEHeader) != SSEAlgorithmAES256 {
|
||||
err = ErrInvalidEncryptionMethod
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,56 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var isRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
}{
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true}, // 0
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true}, // 2
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false}, // 3
|
||||
}
|
||||
|
||||
func TestS3IsRequested(t *testing.T) {
|
||||
for i, test := range isRequestedTests {
|
||||
if got := S3.IsRequested(test.Header); got != test.Expected {
|
||||
t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var parseTests = []struct {
|
||||
Header http.Header
|
||||
ExpectedErr error
|
||||
}{
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil}, // 0
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 2
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 3
|
||||
}
|
||||
|
||||
func TestS3Parse(t *testing.T) {
|
||||
for i, test := range parseTests {
|
||||
if err := S3.Parse(test.Header); err != test.ExpectedErr {
|
||||
t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/hmac"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"io"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
// ObjectKey is a 256 bit secret key used to encrypt the object.
|
||||
// It must never be stored in plaintext.
|
||||
type ObjectKey [32]byte
|
||||
|
||||
// GenerateKey generates a unique ObjectKey from a 256 bit external key
|
||||
// and a source of randomness. If random is nil the default PRNG of system
|
||||
// (crypto/rand) is used.
|
||||
func GenerateKey(extKey [32]byte, random io.Reader) (key ObjectKey) {
|
||||
if random == nil {
|
||||
random = rand.Reader
|
||||
}
|
||||
var nonce [32]byte
|
||||
if _, err := io.ReadFull(random, nonce[:]); err != nil {
|
||||
logger.CriticalIf(context.Background(), errors.New("Unable to read enough randomness from the system"))
|
||||
}
|
||||
sha := sha256.New()
|
||||
sha.Write(extKey[:])
|
||||
sha.Write(nonce[:])
|
||||
sha.Sum(key[:0])
|
||||
return
|
||||
}
|
||||
|
||||
// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed
|
||||
// key is also cryptographically bound to the object's path (bucket/object).
|
||||
func (key ObjectKey) Seal(extKey, iv [32]byte, bucket, object string) []byte {
|
||||
var sealedKey bytes.Buffer
|
||||
mac := hmac.New(sha256.New, extKey[:])
|
||||
mac.Write(iv[:])
|
||||
mac.Write([]byte(filepath.Join(bucket, object)))
|
||||
|
||||
if n, err := sio.Encrypt(&sealedKey, bytes.NewReader(key[:]), sio.Config{Key: mac.Sum(nil)}); n != 64 || err != nil {
|
||||
logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key"))
|
||||
}
|
||||
return sealedKey.Bytes()
|
||||
}
|
||||
|
||||
// Unseal decrypts a sealed key using the 256 bit external key and IV. Since the sealed key
|
||||
// is cryptographically bound to the object's path the same bucket/object as during sealing
|
||||
// must be provided. On success the ObjectKey contains the decrypted sealed key.
|
||||
func (key *ObjectKey) Unseal(sealedKey []byte, extKey, iv [32]byte, bucket, object string) error {
|
||||
var unsealedKey bytes.Buffer
|
||||
mac := hmac.New(sha256.New, extKey[:])
|
||||
mac.Write(iv[:])
|
||||
mac.Write([]byte(filepath.Join(bucket, object)))
|
||||
|
||||
if n, err := sio.Decrypt(&unsealedKey, bytes.NewReader(sealedKey), sio.Config{Key: mac.Sum(nil)}); n != 32 || err != nil {
|
||||
return err // TODO(aead): upgrade sio to use sio.Error
|
||||
}
|
||||
copy(key[:], unsealedKey.Bytes())
|
||||
return nil
|
||||
}
|
||||
|
||||
// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index.
|
||||
func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
|
||||
var bin [4]byte
|
||||
binary.LittleEndian.PutUint32(bin[:], id)
|
||||
|
||||
mac := hmac.New(sha256.New, key[:])
|
||||
mac.Write(bin[:])
|
||||
mac.Sum(partKey[:0])
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) }
|
||||
|
||||
func recoverTest(i int, shouldPass bool, t *testing.T) {
|
||||
if err := recover(); err == nil && !shouldPass {
|
||||
t.Errorf("Test %d should fail but passed successfully", i)
|
||||
} else if err != nil && shouldPass {
|
||||
t.Errorf("Test %d should pass but failed: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
var generateKeyTests = []struct {
|
||||
ExtKey [32]byte
|
||||
Random io.Reader
|
||||
ShouldPass bool
|
||||
}{
|
||||
{ExtKey: [32]byte{}, Random: nil, ShouldPass: true}, // 0
|
||||
{ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true}, // 1
|
||||
{ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2
|
||||
// {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 See: https://github.com/minio/minio/issues/6064
|
||||
}
|
||||
|
||||
func TestGenerateKey(t *testing.T) {
|
||||
for i, test := range generateKeyTests {
|
||||
func() {
|
||||
defer recoverTest(i, test.ShouldPass, t)
|
||||
key := GenerateKey(test.ExtKey, test.Random)
|
||||
if [32]byte(key) == [32]byte{} {
|
||||
t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
var sealUnsealKeyTests = []struct {
|
||||
SealExtKey, SealIV [32]byte
|
||||
SealBucket, SealObject string
|
||||
|
||||
UnsealExtKey, UnsealIV [32]byte
|
||||
UnsealBucket, UnsealObject string
|
||||
|
||||
ShouldPass bool
|
||||
}{
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "object",
|
||||
ShouldPass: true,
|
||||
}, // 0
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{1}, UnsealIV: [32]byte{0}, UnsealBucket: "bucket", UnsealObject: "object",
|
||||
ShouldPass: false,
|
||||
}, // 1
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{1}, UnsealBucket: "bucket", UnsealObject: "object",
|
||||
ShouldPass: false,
|
||||
}, // 2
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "Bucket", UnsealObject: "object",
|
||||
ShouldPass: false,
|
||||
}, // 3
|
||||
{
|
||||
SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealBucket: "bucket", SealObject: "object",
|
||||
UnsealExtKey: [32]byte{}, UnsealIV: [32]byte{}, UnsealBucket: "bucket", UnsealObject: "Object",
|
||||
ShouldPass: false,
|
||||
}, // 4
|
||||
}
|
||||
|
||||
func TestSealUnsealKey(t *testing.T) {
|
||||
for i, test := range sealUnsealKeyTests {
|
||||
key := GenerateKey(test.SealExtKey, rand.Reader)
|
||||
sealedKey := key.Seal(test.SealExtKey, test.SealIV, test.SealBucket, test.SealObject)
|
||||
if err := key.Unseal(sealedKey, test.UnsealExtKey, test.UnsealIV, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass {
|
||||
t.Errorf("Test %d should fail but passed successfully", i)
|
||||
} else if err != nil && test.ShouldPass {
|
||||
t.Errorf("Test %d should pass put failed: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var derivePartKeyTest = []struct {
|
||||
PartID uint32
|
||||
PartKey string
|
||||
}{
|
||||
{PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"},
|
||||
{PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"},
|
||||
{PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"},
|
||||
}
|
||||
|
||||
func TestDerivePartKey(t *testing.T) {
|
||||
var key ObjectKey
|
||||
for i, test := range derivePartKeyTest {
|
||||
expectedPartKey, err := hex.DecodeString(test.PartKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d failed to decode expected part-key: %v", i, err)
|
||||
}
|
||||
partKey := key.DerivePartKey(test.PartID)
|
||||
if !bytes.Equal(partKey[:], expectedPartKey[:]) {
|
||||
t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Minio Cloud Storage, (C) 2015, 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
const (
|
||||
// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
|
||||
S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
|
||||
// S3KMSKeyID is the metadata key referencing the KMS key-id used to
|
||||
// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
|
||||
S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
|
||||
// S3KMSSealedKey is the metadata key referencing the encrypted key generated
|
||||
// by KMS. It is only used for SSE-S3 + KMS.
|
||||
S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
|
||||
)
|
||||
|
||||
// EncryptSinglePart encrypts an io.Reader which must be the
|
||||
// the body of a single-part PUT request.
|
||||
func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
|
||||
r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]})
|
||||
if err != nil {
|
||||
logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key"))
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
// DecryptSinglePart decrypts an io.Writer which must an object
|
||||
// uploaded with the single-part PUT API. The offset and length
|
||||
// specify the requested range.
|
||||
func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser {
|
||||
const PayloadSize = 1 << 16 // DARE 2.0
|
||||
w = ioutil.LimitedWriter(w, offset%PayloadSize, length)
|
||||
|
||||
decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]})
|
||||
if err != nil {
|
||||
logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key"))
|
||||
}
|
||||
return decWriter
|
||||
}
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
type CacheConfig struct {
|
||||
Drives []string `json:"drives"`
|
||||
Expiry int `json:"expiry"`
|
||||
MaxUse int `json:"maxuse"`
|
||||
Exclude []string `json:"exclude"`
|
||||
}
|
||||
|
||||
|
||||
+2
-3
@@ -44,7 +44,6 @@ const (
|
||||
// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
|
||||
cacheSizeMultiplier = 100
|
||||
cacheTrashDir = "trash"
|
||||
cacheMaxDiskUsagePct = 80 // in %
|
||||
cacheCleanupInterval = 10 // in minutes
|
||||
)
|
||||
|
||||
@@ -239,7 +238,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
|
||||
pipeWriter.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
|
||||
pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
|
||||
}()
|
||||
err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo))
|
||||
if err != nil {
|
||||
@@ -839,7 +838,7 @@ func newCache(config CacheConfig) (*diskCache, error) {
|
||||
if err := checkAtimeSupport(dir); err != nil {
|
||||
return nil, errors.New("Atime support required for disk caching")
|
||||
}
|
||||
cache, err := newCacheFSObjects(dir, config.Expiry, cacheMaxDiskUsagePct)
|
||||
cache, err := newCacheFSObjects(dir, config.Expiry, config.MaxUse)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
+126
-6
@@ -28,10 +28,10 @@ import (
|
||||
)
|
||||
|
||||
// Initialize cache FS objects.
|
||||
func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
|
||||
func initCacheFSObjects(disk string, cacheMaxUse int, t *testing.T) (*cacheFSObjects, error) {
|
||||
newTestConfig(globalMinioDefaultRegion)
|
||||
var err error
|
||||
obj, err := newCacheFSObjects(disk, globalCacheExpiry, 100)
|
||||
obj, err := newCacheFSObjects(disk, globalCacheExpiry, cacheMaxUse)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -39,10 +39,10 @@ func initCacheFSObjects(disk string, t *testing.T) (*cacheFSObjects, error) {
|
||||
}
|
||||
|
||||
// inits diskCache struct for nDisks
|
||||
func initDiskCaches(drives []string, t *testing.T) (*diskCache, error) {
|
||||
func initDiskCaches(drives []string, cacheMaxUse int, t *testing.T) (*diskCache, error) {
|
||||
var cfs []*cacheFSObjects
|
||||
for _, d := range drives {
|
||||
obj, err := initCacheFSObjects(d, t)
|
||||
obj, err := initCacheFSObjects(d, cacheMaxUse, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -59,7 +59,46 @@ func TestGetCacheFS(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := initDiskCaches(fsDirs, t)
|
||||
d, err := initDiskCaches(fsDirs, 100, t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
bucketName := "testbucket"
|
||||
objectName := "testobject"
|
||||
ctx := context.Background()
|
||||
// find cache drive where object would be hashed
|
||||
index := d.hashIndex(bucketName, objectName)
|
||||
// turn off drive by setting online status to false
|
||||
d.cfs[index].online = false
|
||||
cfs, err := d.getCacheFS(ctx, bucketName, objectName)
|
||||
if n == 1 && err == errDiskNotFound {
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
i := -1
|
||||
for j, f := range d.cfs {
|
||||
if f == cfs {
|
||||
i = j
|
||||
break
|
||||
}
|
||||
}
|
||||
if i != (index+1)%n {
|
||||
t.Fatalf("expected next cache location to be picked")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// test whether a drive being offline causes
|
||||
// getCacheFS to fetch next online drive
|
||||
func TestGetCacheFSMaxUse(t *testing.T) {
|
||||
for n := 1; n < 10; n++ {
|
||||
fsDirs, err := getRandomDisks(n)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -141,7 +180,7 @@ func TestDiskCache(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := initDiskCaches(fsDirs, t)
|
||||
d, err := initDiskCaches(fsDirs, 100, t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -209,6 +248,87 @@ func TestDiskCache(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test diskCache with upper bound on max cache use.
|
||||
func TestDiskCacheMaxUse(t *testing.T) {
|
||||
fsDirs, err := getRandomDisks(1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
d, err := initDiskCaches(fsDirs, globalCacheMaxUse, t)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cache := d.cfs[0]
|
||||
ctx := context.Background()
|
||||
bucketName := "testbucket"
|
||||
objectName := "testobject"
|
||||
content := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
etag := "061208c10af71a30c6dcd6cf5d89f0fe"
|
||||
contentType := "application/zip"
|
||||
size := len(content)
|
||||
|
||||
httpMeta := make(map[string]string)
|
||||
httpMeta["etag"] = etag
|
||||
httpMeta["content-type"] = contentType
|
||||
|
||||
objInfo := ObjectInfo{}
|
||||
objInfo.Bucket = bucketName
|
||||
objInfo.Name = objectName
|
||||
objInfo.Size = int64(size)
|
||||
objInfo.ContentType = contentType
|
||||
objInfo.ETag = etag
|
||||
objInfo.UserDefined = httpMeta
|
||||
|
||||
byteReader := bytes.NewReader([]byte(content))
|
||||
hashReader, err := hash.NewReader(byteReader, int64(size), "", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cache.diskAvailable(int64(size)) {
|
||||
err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
|
||||
if err != errDiskFull {
|
||||
t.Fatal("Cache max-use limit violated.")
|
||||
}
|
||||
} else {
|
||||
err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cachedObjInfo, err := cache.GetObjectInfo(ctx, bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cache.Exists(ctx, bucketName, objectName) {
|
||||
t.Fatal("Expected object to exist on cache")
|
||||
}
|
||||
if cachedObjInfo.ETag != objInfo.ETag {
|
||||
t.Fatal("Expected ETag to match")
|
||||
}
|
||||
if cachedObjInfo.Size != objInfo.Size {
|
||||
t.Fatal("Size mismatch")
|
||||
}
|
||||
if cachedObjInfo.ContentType != objInfo.ContentType {
|
||||
t.Fatal("Cached content-type does not match")
|
||||
}
|
||||
writer := bytes.NewBuffer(nil)
|
||||
err = cache.Get(ctx, bucketName, objectName, 0, int64(size), writer, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if ccontent := writer.Bytes(); !bytes.Equal([]byte(content), ccontent) {
|
||||
t.Errorf("wrong cached file content")
|
||||
}
|
||||
err = cache.Delete(ctx, bucketName, objectName)
|
||||
if err != nil {
|
||||
t.Errorf("object missing from cache")
|
||||
}
|
||||
online := cache.IsOnline()
|
||||
if !online {
|
||||
t.Errorf("expected cache drive to be online")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsCacheExcludeDirective(t *testing.T) {
|
||||
testCases := []struct {
|
||||
cacheControlOpt string
|
||||
|
||||
@@ -178,7 +178,6 @@ func fsStatVolume(ctx context.Context, volume string) (os.FileInfo, error) {
|
||||
}
|
||||
|
||||
if !fi.IsDir() {
|
||||
logger.LogIf(ctx, errVolumeAccessDenied)
|
||||
return nil, errVolumeAccessDenied
|
||||
}
|
||||
|
||||
@@ -200,7 +199,7 @@ func osErrToFSFileErr(err error) error {
|
||||
return errFileAccessDenied
|
||||
}
|
||||
if isSysErrNotDir(err) {
|
||||
return errFileAccessDenied
|
||||
return errFileNotFound
|
||||
}
|
||||
if isSysErrPathNotFound(err) {
|
||||
return errFileNotFound
|
||||
@@ -219,8 +218,7 @@ func fsStatDir(ctx context.Context, statDir string) (os.FileInfo, error) {
|
||||
return nil, err
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
logger.LogIf(ctx, errFileAccessDenied)
|
||||
return nil, errFileAccessDenied
|
||||
return nil, errFileNotFound
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
@@ -245,8 +243,7 @@ func fsStatFile(ctx context.Context, statFile string) (os.FileInfo, error) {
|
||||
return nil, err
|
||||
}
|
||||
if fi.IsDir() {
|
||||
logger.LogIf(ctx, errFileAccessDenied)
|
||||
return nil, errFileAccessDenied
|
||||
return nil, errFileNotFound
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
@@ -136,7 +136,7 @@ func TestFSStats(t *testing.T) {
|
||||
srcFSPath: path,
|
||||
srcVol: "success-vol",
|
||||
srcPath: "path",
|
||||
expectedErr: errFileAccessDenied,
|
||||
expectedErr: errFileNotFound,
|
||||
},
|
||||
// Test case - 6.
|
||||
// Test case with src path segment > 255.
|
||||
|
||||
@@ -264,7 +264,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
|
||||
}
|
||||
return
|
||||
}
|
||||
// Close writer explicitly signalling we wrote all data.
|
||||
// Close writer explicitly signaling we wrote all data.
|
||||
if gerr := srcInfo.Writer.Close(); gerr != nil {
|
||||
logger.LogIf(ctx, gerr)
|
||||
return
|
||||
|
||||
+63
-37
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
"github.com/minio/minio/pkg/mountinfo"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
@@ -44,8 +45,6 @@ var defaultEtag = "00000000000000000000000000000000-1"
|
||||
type FSObjects struct {
|
||||
// Disk usage metrics
|
||||
totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
// Disk usage running routine
|
||||
usageRunning int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
|
||||
// Path to be exported over S3 API.
|
||||
fsPath string
|
||||
@@ -64,6 +63,8 @@ type FSObjects struct {
|
||||
// ListObjects pool management.
|
||||
listPool *treeWalkPool
|
||||
|
||||
diskMount bool
|
||||
|
||||
appendFileMap map[string]*fsAppendFile
|
||||
appendFileMapMu sync.Mutex
|
||||
|
||||
@@ -109,6 +110,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
|
||||
var err error
|
||||
if fsPath, err = getValidPath(fsPath); err != nil {
|
||||
if err == errMinDiskSize {
|
||||
return nil, err
|
||||
}
|
||||
return nil, uiErrUnableToWriteInBackend(err)
|
||||
}
|
||||
|
||||
@@ -138,6 +142,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
nsMutex: newNSLock(false),
|
||||
listPool: newTreeWalkPool(globalLookupTimeout),
|
||||
appendFileMap: make(map[string]*fsAppendFile),
|
||||
diskMount: mountinfo.IsLikelyMountPoint(fsPath),
|
||||
}
|
||||
|
||||
// Once the filesystem has initialized hold the read lock for
|
||||
@@ -156,7 +161,10 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
return nil, uiErrUnableToReadFromBackend(err).Msg("Unable to initialize policy system")
|
||||
}
|
||||
|
||||
go fs.diskUsage(globalServiceDoneCh)
|
||||
if !fs.diskMount {
|
||||
go fs.diskUsage(globalServiceDoneCh)
|
||||
}
|
||||
|
||||
go fs.cleanupStaleMultipartUploads(ctx, globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
|
||||
|
||||
// Return successfully initialized object layer.
|
||||
@@ -173,32 +181,40 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
|
||||
|
||||
// diskUsage returns du information for the posix path, in a continuous routine.
|
||||
func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
ticker := time.NewTicker(globalUsageCheckInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
usageFn := func(ctx context.Context, entry string) error {
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
if globalHTTPServer != nil {
|
||||
// Wait at max 1 minute for an inprogress request
|
||||
// before proceeding to count the usage.
|
||||
waitCount := 60
|
||||
// Any requests in progress, delay the usage.
|
||||
for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
|
||||
waitCount--
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
select {
|
||||
case <-doneCh:
|
||||
return errWalkAbort
|
||||
default:
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
|
||||
}
|
||||
atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Check if disk usage routine is running, if yes then return.
|
||||
if atomic.LoadInt32(&fs.usageRunning) == 1 {
|
||||
return
|
||||
}
|
||||
atomic.StoreInt32(&fs.usageRunning, 1)
|
||||
defer atomic.StoreInt32(&fs.usageRunning, 0)
|
||||
|
||||
if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err != nil {
|
||||
// Return this routine upon errWalkAbort, continue for any other error on purpose
|
||||
// so that we can start the routine freshly in another 12 hours.
|
||||
if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err == errWalkAbort {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -206,15 +222,20 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
select {
|
||||
case <-doneCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
// Check if disk usage routine is running, if yes let it finish.
|
||||
if atomic.LoadInt32(&fs.usageRunning) == 1 {
|
||||
continue
|
||||
}
|
||||
atomic.StoreInt32(&fs.usageRunning, 1)
|
||||
|
||||
case <-time.After(globalUsageCheckInterval):
|
||||
var usage uint64
|
||||
usageFn = func(ctx context.Context, entry string) error {
|
||||
if globalHTTPServer != nil {
|
||||
// Wait at max 1 minute for an inprogress request
|
||||
// before proceeding to count the usage.
|
||||
waitCount := 60
|
||||
// Any requests in progress, delay the usage.
|
||||
for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
|
||||
waitCount--
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
@@ -230,11 +251,8 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
}
|
||||
|
||||
if err := getDiskUsage(context.Background(), fs.fsPath, usageFn); err != nil {
|
||||
atomic.StoreInt32(&fs.usageRunning, 0)
|
||||
continue
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&fs.usageRunning, 0)
|
||||
atomic.StoreUint64(&fs.totalUsed, usage)
|
||||
}
|
||||
}
|
||||
@@ -242,8 +260,16 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
|
||||
// StorageInfo - returns underlying storage statistics.
|
||||
func (fs *FSObjects) StorageInfo(ctx context.Context) StorageInfo {
|
||||
di, err := getDiskInfo(fs.fsPath)
|
||||
if err != nil {
|
||||
return StorageInfo{}
|
||||
}
|
||||
used := di.Total - di.Free
|
||||
if !fs.diskMount {
|
||||
used = atomic.LoadUint64(&fs.totalUsed)
|
||||
}
|
||||
storageInfo := StorageInfo{
|
||||
Used: atomic.LoadUint64(&fs.totalUsed),
|
||||
Used: used,
|
||||
}
|
||||
storageInfo.Backend.Type = FS
|
||||
return storageInfo
|
||||
@@ -479,7 +505,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
|
||||
}
|
||||
return
|
||||
}
|
||||
// Close writer explicitly signalling we wrote all data.
|
||||
// Close writer explicitly signaling we wrote all data.
|
||||
if gerr := srcInfo.Writer.Close(); gerr != nil {
|
||||
logger.LogIf(ctx, gerr)
|
||||
return
|
||||
@@ -695,7 +721,7 @@ func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object s
|
||||
}
|
||||
|
||||
if _, err := fs.statBucketDir(ctx, bucket); err != nil {
|
||||
return oi, toObjectErr(err, bucket)
|
||||
return oi, err
|
||||
}
|
||||
|
||||
if strings.HasSuffix(object, slashSeparator) && !fs.isObjectDir(bucket, object) {
|
||||
@@ -1214,7 +1240,7 @@ func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy
|
||||
|
||||
// GetBucketPolicy will get policy on bucket
|
||||
func (fs *FSObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
|
||||
return GetPolicyConfig(fs, bucket)
|
||||
return getPolicyConfig(fs, bucket)
|
||||
}
|
||||
|
||||
// DeleteBucketPolicy deletes all policies on bucket
|
||||
|
||||
@@ -154,6 +154,7 @@ func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
|
||||
UserDefined: userDefined,
|
||||
ContentType: oi.ContentType,
|
||||
ContentEncoding: oi.Metadata.Get("Content-Encoding"),
|
||||
StorageClass: oi.StorageClass,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+9
-2
@@ -34,7 +34,7 @@ import (
|
||||
)
|
||||
|
||||
func init() {
|
||||
logger.Init(GOPATH)
|
||||
logger.Init(GOPATH, GOROOT)
|
||||
logger.RegisterUIError(fmtError)
|
||||
}
|
||||
|
||||
@@ -197,7 +197,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
getCert = globalTLSCerts.GetCertificate
|
||||
}
|
||||
|
||||
globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, registerHandlers(router, globalHandlers...), getCert)
|
||||
globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
|
||||
globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
|
||||
globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
|
||||
go func() {
|
||||
@@ -215,6 +215,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
logger.FatalIf(err, "Unable to initialize gateway backend")
|
||||
}
|
||||
|
||||
if gw.Name() != "nas" {
|
||||
// Initialize policy sys for all gateways. NAS gateway already
|
||||
// initializes policy sys internally, avoid double initialization.
|
||||
// Additionally also don't block the initialization of gateway.
|
||||
go globalPolicySys.Init(newObject)
|
||||
}
|
||||
|
||||
// Once endpoints are finalized, initialize the new object api.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = newObject
|
||||
|
||||
@@ -82,6 +82,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend.
|
||||
@@ -100,6 +101,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
`
|
||||
|
||||
@@ -238,6 +240,8 @@ func s3MetaToAzureProperties(ctx context.Context, s3Metadata map[string]string)
|
||||
props.ContentMD5 = v
|
||||
case k == "Content-Type":
|
||||
props.ContentType = v
|
||||
case k == "Content-Language":
|
||||
props.ContentLanguage = v
|
||||
}
|
||||
}
|
||||
return blobMeta, props, nil
|
||||
@@ -292,6 +296,9 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper
|
||||
if props.ContentType != "" {
|
||||
s3Metadata["Content-Type"] = props.ContentType
|
||||
}
|
||||
if props.ContentLanguage != "" {
|
||||
s3Metadata["Content-Language"] = props.ContentLanguage
|
||||
}
|
||||
return s3Metadata
|
||||
}
|
||||
|
||||
|
||||
@@ -72,6 +72,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for B2 backend.
|
||||
@@ -85,6 +86,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
`
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
|
||||
@@ -118,6 +118,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
GCS credentials file:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json
|
||||
@@ -137,6 +138,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}} mygcsprojectid
|
||||
`
|
||||
|
||||
|
||||
@@ -78,6 +78,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Manta Object Storage backend.
|
||||
@@ -102,6 +103,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
`
|
||||
|
||||
|
||||
@@ -17,12 +17,9 @@
|
||||
package nas
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/minio/cli"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -57,6 +54,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for NAS backend.
|
||||
@@ -70,6 +68,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}} /shared/nasvol
|
||||
`
|
||||
|
||||
@@ -126,8 +125,3 @@ type nasObjects struct {
|
||||
func (l *nasObjects) IsNotificationSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// GetBucketPolicy will get policy on bucket
|
||||
func (l *nasObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
|
||||
return minio.GetPolicyConfig(l, bucket)
|
||||
}
|
||||
|
||||
@@ -74,6 +74,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Aliyun OSS backend.
|
||||
@@ -92,6 +93,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
`
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -65,6 +65,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend.
|
||||
@@ -83,6 +84,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
`
|
||||
|
||||
@@ -97,12 +99,16 @@ EXAMPLES:
|
||||
|
||||
// Handler for 'minio gateway s3' command line.
|
||||
func s3GatewayMain(ctx *cli.Context) {
|
||||
// Validate gateway arguments.
|
||||
host := ctx.Args().First()
|
||||
// Validate gateway arguments.
|
||||
logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||
args := ctx.Args()
|
||||
if !ctx.Args().Present() {
|
||||
args = cli.Args{"https://s3.amazonaws.com"}
|
||||
}
|
||||
|
||||
minio.StartGateway(ctx, &S3{host})
|
||||
// Validate gateway arguments.
|
||||
logger.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), args.First()), "Invalid argument")
|
||||
|
||||
// Start the gateway..
|
||||
minio.StartGateway(ctx, &S3{args.First()})
|
||||
}
|
||||
|
||||
// S3 implements Gateway.
|
||||
@@ -115,34 +121,46 @@ func (g *S3) Name() string {
|
||||
return s3Backend
|
||||
}
|
||||
|
||||
// NewGatewayLayer returns s3 ObjectLayer.
|
||||
func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
|
||||
var err error
|
||||
var endpoint string
|
||||
var secure = true
|
||||
// newS3 - Initializes a new client by auto probing S3 server signature.
|
||||
func newS3(url, accessKey, secretKey string) (*miniogo.Core, error) {
|
||||
if url == "" {
|
||||
url = "https://s3.amazonaws.com"
|
||||
}
|
||||
|
||||
// Validate host parameters.
|
||||
if g.host != "" {
|
||||
// Override default params if the host is provided
|
||||
endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
|
||||
// Override default params if the host is provided
|
||||
endpoint, secure, err := minio.ParseGatewayEndpoint(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
clnt, err := miniogo.NewV4(endpoint, accessKey, secretKey, secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
|
||||
clnt, err = miniogo.NewV2(endpoint, accessKey, secretKey, secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if _, err = clnt.BucketExists("probe-bucket-sign"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Default endpoint parameters
|
||||
if endpoint == "" {
|
||||
endpoint = "s3.amazonaws.com"
|
||||
}
|
||||
return &miniogo.Core{Client: clnt}, nil
|
||||
}
|
||||
|
||||
// Initialize minio client object.
|
||||
client, err := miniogo.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
|
||||
// NewGatewayLayer returns s3 ObjectLayer.
|
||||
func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
|
||||
// Probe S3 signature with input credentials.
|
||||
clnt, err := newS3(g.host, creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &s3Objects{
|
||||
Client: client,
|
||||
Client: clnt,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -255,7 +273,7 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
|
||||
|
||||
// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
|
||||
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
|
||||
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys, startAfter)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return loi, minio.ErrorRespToObjectError(err, bucket)
|
||||
|
||||
@@ -79,6 +79,7 @@ ENVIRONMENT VARIABLES: (Default values in parenthesis)
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
SIA_TEMP_DIR: The name of the local Sia temporary storage directory. (.sia_temp)
|
||||
SIA_API_PASSWORD: API password for Sia daemon. (default is empty)
|
||||
@@ -91,6 +92,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}}
|
||||
`
|
||||
|
||||
|
||||
+19
-2
@@ -688,10 +688,10 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
|
||||
// rate.Limiter token bucket configured with maxOpenFileLimit and
|
||||
// burst set to 1. The request will idle for up to 1*time.Second.
|
||||
// If the limiter detects the deadline will be exceeded, the request is
|
||||
// cancelled immediately.
|
||||
// canceled immediately.
|
||||
func setRateLimitHandler(h http.Handler) http.Handler {
|
||||
_, maxLimit, err := sys.GetMaxOpenFileLimit()
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to get maximum open file limit", context.Background())
|
||||
// Burst value is set to 1 to allow only maxOpenFileLimit
|
||||
// requests to happen at once.
|
||||
l := rate.NewLimiter(rate.Limit(maxLimit), 1)
|
||||
@@ -738,3 +738,20 @@ func (s securityHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request)
|
||||
header.Set("Content-Security-Policy", "block-all-mixed-content") // prevent mixed (HTTP / HTTPS content)
|
||||
s.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// criticalErrorHandler handles critical server failures caused by
|
||||
// `panic(logger.ErrCritical)` as done by `logger.CriticalIf`.
|
||||
//
|
||||
// It should be always the first / highest HTTP handler.
|
||||
type criticalErrorHandler struct{ handler http.Handler }
|
||||
|
||||
func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err == logger.ErrCritical { // handle
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
} else if err != nil {
|
||||
panic(err) // forward other panic calls
|
||||
}
|
||||
}()
|
||||
h.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
@@ -194,6 +194,8 @@ var (
|
||||
|
||||
// Disk cache expiry
|
||||
globalCacheExpiry = 90
|
||||
// Max allowed disk cache percentage
|
||||
globalCacheMaxUse = 80
|
||||
|
||||
// RPC V1 - Initial version
|
||||
// RPC V2 - format.json XL version changed to 2
|
||||
|
||||
+6
-1
@@ -59,7 +59,12 @@ type Server struct {
|
||||
listenerMutex *sync.Mutex // to guard 'listener' field.
|
||||
listener *httpListener // HTTP listener for all 'Addrs' field.
|
||||
inShutdown uint32 // indicates whether the server is in shutdown or not
|
||||
requestCount int32 // counter holds no. of request in process.
|
||||
requestCount int32 // counter holds no. of request in progress.
|
||||
}
|
||||
|
||||
// GetRequestCount - returns number of request in progress.
|
||||
func (srv *Server) GetRequestCount() int32 {
|
||||
return atomic.LoadInt32(&srv.requestCount)
|
||||
}
|
||||
|
||||
// Start - start HTTP server
|
||||
|
||||
@@ -124,7 +124,10 @@ func (l *lockRPCReceiver) lockMaintenance(interval time.Duration) {
|
||||
for _, nlrip := range nlripLongLived {
|
||||
// Initialize client based on the long live locks.
|
||||
host, err := xnet.ParseHost(nlrip.lri.node)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
continue
|
||||
}
|
||||
c, err := NewLockRPCClient(host)
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
@@ -183,7 +186,7 @@ func NewLockRPCServer() (*xrpc.Server, error) {
|
||||
// Register distributed NS lock handlers.
|
||||
func registerDistNSLockRouter(router *mux.Router) {
|
||||
rpcServer, err := NewLockRPCServer()
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to initialize Lock RPC Server", context.Background())
|
||||
|
||||
// Start lock maintenance from all lock servers.
|
||||
go startLockMaintenance(globalLockServer)
|
||||
|
||||
+47
-13
@@ -52,6 +52,25 @@ var matchingFuncNames = [...]string{
|
||||
"http.HandlerFunc.ServeHTTP",
|
||||
"cmd.serverMain",
|
||||
"cmd.StartGateway",
|
||||
"cmd.(*webAPIHandlers).ListBuckets",
|
||||
"cmd.(*webAPIHandlers).MakeBucket",
|
||||
"cmd.(*webAPIHandlers).DeleteBucket",
|
||||
"cmd.(*webAPIHandlers).ListObjects",
|
||||
"cmd.(*webAPIHandlers).RemoveObject",
|
||||
"cmd.(*webAPIHandlers).Login",
|
||||
"cmd.(*webAPIHandlers).GenerateAuth",
|
||||
"cmd.(*webAPIHandlers).SetAuth",
|
||||
"cmd.(*webAPIHandlers).GetAuth",
|
||||
"cmd.(*webAPIHandlers).CreateURLToken",
|
||||
"cmd.(*webAPIHandlers).Upload",
|
||||
"cmd.(*webAPIHandlers).Download",
|
||||
"cmd.(*webAPIHandlers).DownloadZip",
|
||||
"cmd.(*webAPIHandlers).GetBucketPolicy",
|
||||
"cmd.(*webAPIHandlers).ListAllBucketPolicies",
|
||||
"cmd.(*webAPIHandlers).SetBucketPolicy",
|
||||
"cmd.(*webAPIHandlers).PresignedGet",
|
||||
"cmd.(*webAPIHandlers).ServerInfo",
|
||||
"cmd.(*webAPIHandlers).StorageInfo",
|
||||
// add more here ..
|
||||
}
|
||||
|
||||
@@ -142,22 +161,24 @@ func RegisterUIError(f func(string, error, bool) string) {
|
||||
// and GOROOT directories. Also append github.com/minio/minio
|
||||
// This is done to clean up the filename, when stack trace is
|
||||
// displayed when an error happens.
|
||||
func Init(goPath string) {
|
||||
func Init(goPath string, goRoot string) {
|
||||
|
||||
var goPathList []string
|
||||
var goRootList []string
|
||||
var defaultgoPathList []string
|
||||
var defaultgoRootList []string
|
||||
pathSeperator := ":"
|
||||
// Add all possible GOPATH paths into trimStrings
|
||||
// Split GOPATH depending on the OS type
|
||||
if runtime.GOOS == "windows" {
|
||||
goPathList = strings.Split(goPath, ";")
|
||||
defaultgoPathList = strings.Split(build.Default.GOPATH, ";")
|
||||
} else {
|
||||
// All other types of OSs
|
||||
goPathList = strings.Split(goPath, ":")
|
||||
defaultgoPathList = strings.Split(build.Default.GOPATH, ":")
|
||||
|
||||
pathSeperator = ";"
|
||||
}
|
||||
|
||||
goPathList = strings.Split(goPath, pathSeperator)
|
||||
goRootList = strings.Split(goRoot, pathSeperator)
|
||||
defaultgoPathList = strings.Split(build.Default.GOPATH, pathSeperator)
|
||||
defaultgoRootList = strings.Split(build.Default.GOROOT, pathSeperator)
|
||||
|
||||
// Add trim string "{GOROOT}/src/" into trimStrings
|
||||
trimStrings = []string{filepath.Join(runtime.GOROOT(), "src") + string(filepath.Separator)}
|
||||
|
||||
@@ -167,10 +188,21 @@ func Init(goPath string) {
|
||||
trimStrings = append(trimStrings, filepath.Join(goPathString, "src")+string(filepath.Separator))
|
||||
}
|
||||
|
||||
for _, goRootString := range goRootList {
|
||||
trimStrings = append(trimStrings, filepath.Join(goRootString, "src")+string(filepath.Separator))
|
||||
}
|
||||
|
||||
for _, defaultgoPathString := range defaultgoPathList {
|
||||
trimStrings = append(trimStrings, filepath.Join(defaultgoPathString, "src")+string(filepath.Separator))
|
||||
}
|
||||
|
||||
for _, defaultgoRootString := range defaultgoRootList {
|
||||
trimStrings = append(trimStrings, filepath.Join(defaultgoRootString, "src")+string(filepath.Separator))
|
||||
}
|
||||
|
||||
// Remove duplicate entries.
|
||||
trimStrings = uniqueEntries(trimStrings)
|
||||
|
||||
// Add "github.com/minio/minio" as the last to cover
|
||||
// paths like "{GOROOT}/src/github.com/minio/minio"
|
||||
// and "{GOPATH}/src/github.com/minio/minio"
|
||||
@@ -200,7 +232,7 @@ func getTrace(traceLevel int) []string {
|
||||
var trace []string
|
||||
pc, file, lineNumber, ok := runtime.Caller(traceLevel)
|
||||
|
||||
for ok {
|
||||
for ok && file != "" {
|
||||
// Clean up the common prefixes
|
||||
file = trimTrace(file)
|
||||
// Get the function name
|
||||
@@ -330,13 +362,15 @@ func LogIf(ctx context.Context, err error) {
|
||||
fmt.Println(output)
|
||||
}
|
||||
|
||||
// CriticalIf :
|
||||
// Like LogIf with exit
|
||||
// It'll be called for fatal error conditions during run-time
|
||||
// ErrCritical is the value panic'd whenever CriticalIf is called.
|
||||
var ErrCritical struct{}
|
||||
|
||||
// CriticalIf logs the provided error on the console. It fails the
|
||||
// current go-routine by causing a `panic(ErrCritical)`.
|
||||
func CriticalIf(ctx context.Context, err error) {
|
||||
if err != nil {
|
||||
LogIf(ctx, err)
|
||||
os.Exit(1)
|
||||
panic(ErrCritical)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,3 +50,16 @@ func ansiSaveAttributes() {
|
||||
func ansiRestoreAttributes() {
|
||||
ansiEscape("8")
|
||||
}
|
||||
|
||||
func uniqueEntries(paths []string) []string {
|
||||
found := map[string]bool{}
|
||||
unqiue := []string{}
|
||||
|
||||
for v := range paths {
|
||||
if _, ok := found[paths[v]]; !ok {
|
||||
found[paths[v]] = true
|
||||
unqiue = append(unqiue, paths[v])
|
||||
}
|
||||
}
|
||||
return unqiue
|
||||
}
|
||||
|
||||
@@ -86,9 +86,9 @@ func newDsyncNodes(endpoints EndpointList) (clnts []dsync.NetLocker, myNode int)
|
||||
locker = &(receiver.ll)
|
||||
} else {
|
||||
host, err := xnet.ParseHost(endpoint.Host)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to parse Lock RPC Host", context.Background())
|
||||
locker, err = NewLockRPCClient(host)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to initialize Lock RPC Client", context.Background())
|
||||
}
|
||||
|
||||
clnts = append(clnts, locker)
|
||||
|
||||
@@ -226,7 +226,7 @@ func TestNamespaceForceUnlockTest(t *testing.T) {
|
||||
|
||||
select {
|
||||
case <-ch:
|
||||
// Signalled so all is fine.
|
||||
// Signaled so all is fine.
|
||||
break
|
||||
|
||||
case <-time.After(100 * time.Millisecond):
|
||||
|
||||
@@ -166,9 +166,9 @@ func makeRemoteRPCClients(endpoints EndpointList) map[xnet.Host]*PeerRPCClient {
|
||||
peerRPCClientMap := make(map[xnet.Host]*PeerRPCClient)
|
||||
for _, hostStr := range GetRemotePeers(endpoints) {
|
||||
host, err := xnet.ParseHost(hostStr)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to parse peer RPC Host", context.Background())
|
||||
rpcClient, err := NewPeerRPCClient(host)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to parse peer RPC Client", context.Background())
|
||||
peerRPCClientMap[*host] = rpcClient
|
||||
}
|
||||
|
||||
|
||||
@@ -201,7 +201,7 @@ func NewPeerRPCServer() (*xrpc.Server, error) {
|
||||
// registerPeerRPCRouter - creates and registers Peer RPC server and its router.
|
||||
func registerPeerRPCRouter(router *mux.Router) {
|
||||
rpcServer, err := NewPeerRPCServer()
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to initialize peer RPC Server", context.Background())
|
||||
subrouter := router.PathPrefix(minioReservedBucketPath).Subrouter()
|
||||
subrouter.Path(peerServiceSubPath).Handler(rpcServer)
|
||||
}
|
||||
|
||||
+3
-3
@@ -99,7 +99,7 @@ func (sys *PolicySys) refresh(objAPI ObjectLayer) error {
|
||||
}
|
||||
sys.removeDeletedBuckets(buckets)
|
||||
for _, bucket := range buckets {
|
||||
config, err := GetPolicyConfig(objAPI, bucket.Name)
|
||||
config, err := objAPI.GetBucketPolicy(context.Background(), bucket.Name)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketPolicyNotFound); ok {
|
||||
sys.Remove(bucket.Name)
|
||||
@@ -187,8 +187,8 @@ func getConditionValues(request *http.Request, locationConstraint string) map[st
|
||||
return args
|
||||
}
|
||||
|
||||
// GetPolicyConfig - get policy config for given bucket name.
|
||||
func GetPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
|
||||
// getPolicyConfig - get policy config for given bucket name.
|
||||
func getPolicyConfig(objAPI ObjectLayer, bucketName string) (*policy.Policy, error) {
|
||||
// Construct path to policy.json for the given bucket.
|
||||
configFile := path.Join(bucketConfigPrefix, bucketName, bucketPolicyConfig)
|
||||
|
||||
|
||||
+45
-27
@@ -34,6 +34,7 @@ import (
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/mountinfo"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -60,15 +61,15 @@ func isValidVolname(volname string) bool {
|
||||
// posix - implements StorageAPI interface.
|
||||
type posix struct {
|
||||
// Disk usage metrics
|
||||
totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
// Disk usage running routine
|
||||
usageRunning int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
ioErrCount int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
totalUsed uint64 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
ioErrCount int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
|
||||
|
||||
diskPath string
|
||||
pool sync.Pool
|
||||
connected bool
|
||||
|
||||
diskMount bool // indicates if the path is an actual mount.
|
||||
|
||||
// Disk usage metrics
|
||||
stopUsageCh chan struct{}
|
||||
}
|
||||
@@ -135,10 +136,10 @@ func getValidPath(path string) (string, error) {
|
||||
if err != nil {
|
||||
return path, err
|
||||
}
|
||||
defer os.Remove(pathJoin(path, ".writable-check.tmp"))
|
||||
file.Close()
|
||||
|
||||
err = os.Remove(pathJoin(path, ".writable-check.tmp"))
|
||||
return path, err
|
||||
return path, nil
|
||||
}
|
||||
|
||||
// isDirEmpty - returns whether given directory is empty or not.
|
||||
@@ -183,9 +184,12 @@ func newPosix(path string) (*posix, error) {
|
||||
},
|
||||
},
|
||||
stopUsageCh: make(chan struct{}),
|
||||
diskMount: mountinfo.IsLikelyMountPoint(path),
|
||||
}
|
||||
|
||||
go p.diskUsage(globalServiceDoneCh)
|
||||
if !p.diskMount {
|
||||
go p.diskUsage(globalServiceDoneCh)
|
||||
}
|
||||
|
||||
// Success.
|
||||
return p, nil
|
||||
@@ -218,7 +222,7 @@ func checkDiskMinTotal(di disk.Info) (err error) {
|
||||
// used for journalling, inodes etc.
|
||||
totalDiskSpace := float64(di.Total) * 0.95
|
||||
if int64(totalDiskSpace) <= diskMinTotalSpace {
|
||||
return errDiskFull
|
||||
return errMinDiskSize
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -293,10 +297,14 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
|
||||
if err != nil {
|
||||
return info, err
|
||||
}
|
||||
used := di.Total - di.Free
|
||||
if !s.diskMount {
|
||||
used = atomic.LoadUint64(&s.totalUsed)
|
||||
}
|
||||
return DiskInfo{
|
||||
Total: di.Total,
|
||||
Free: di.Free,
|
||||
Used: atomic.LoadUint64(&s.totalUsed),
|
||||
Used: used,
|
||||
}, nil
|
||||
|
||||
}
|
||||
@@ -336,7 +344,20 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
defer ticker.Stop()
|
||||
|
||||
usageFn := func(ctx context.Context, entry string) error {
|
||||
if globalHTTPServer != nil {
|
||||
// Wait at max 1 minute for an inprogress request
|
||||
// before proceeding to count the usage.
|
||||
waitCount := 60
|
||||
// Any requests in progress, delay the usage.
|
||||
for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
|
||||
waitCount--
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-doneCh:
|
||||
return errWalkAbort
|
||||
case <-s.stopUsageCh:
|
||||
return errWalkAbort
|
||||
default:
|
||||
@@ -349,14 +370,9 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
}
|
||||
}
|
||||
|
||||
// Check if disk usage routine is running, if yes then return.
|
||||
if atomic.LoadInt32(&s.usageRunning) == 1 {
|
||||
return
|
||||
}
|
||||
atomic.StoreInt32(&s.usageRunning, 1)
|
||||
defer atomic.StoreInt32(&s.usageRunning, 0)
|
||||
|
||||
if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err != nil {
|
||||
// Return this routine upon errWalkAbort, continue for any other error on purpose
|
||||
// so that we can start the routine freshly in another 12 hours.
|
||||
if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err == errWalkAbort {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -366,16 +382,20 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
return
|
||||
case <-doneCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
// Check if disk usage routine is running, if yes let it
|
||||
// finish, before starting a new one.
|
||||
if atomic.LoadInt32(&s.usageRunning) == 1 {
|
||||
continue
|
||||
}
|
||||
atomic.StoreInt32(&s.usageRunning, 1)
|
||||
|
||||
case <-time.After(globalUsageCheckInterval):
|
||||
var usage uint64
|
||||
usageFn = func(ctx context.Context, entry string) error {
|
||||
if globalHTTPServer != nil {
|
||||
// Wait at max 1 minute for an inprogress request
|
||||
// before proceeding to count the usage.
|
||||
waitCount := 60
|
||||
// Any requests in progress, delay the usage.
|
||||
for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
|
||||
waitCount--
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
select {
|
||||
case <-s.stopUsageCh:
|
||||
return errWalkAbort
|
||||
@@ -390,11 +410,9 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
}
|
||||
|
||||
if err := getDiskUsage(context.Background(), s.diskPath, usageFn); err != nil {
|
||||
atomic.StoreInt32(&s.usageRunning, 0)
|
||||
continue
|
||||
}
|
||||
|
||||
atomic.StoreInt32(&s.usageRunning, 0)
|
||||
atomic.StoreUint64(&s.totalUsed, usage)
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -470,7 +470,7 @@ func TestPosixMakeVol(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestPosixDeleteVol - Validates the expected behaviour of posix.DeleteVol for various cases.
|
||||
// TestPosixDeleteVol - Validates the expected behavior of posix.DeleteVol for various cases.
|
||||
func TestPosixDeleteVol(t *testing.T) {
|
||||
// create posix test setup
|
||||
posixStorage, path, err := newPosixTestSetup()
|
||||
@@ -1948,7 +1948,7 @@ func TestCheckDiskTotalMin(t *testing.T) {
|
||||
FSType: "XFS",
|
||||
Files: 9999,
|
||||
},
|
||||
err: errDiskFull,
|
||||
err: errMinDiskSize,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -69,7 +69,7 @@ func TestUNCPaths(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test to validate posix behaviour on windows when a non-final path component is a file.
|
||||
// Test to validate posix behavior on windows when a non-final path component is a file.
|
||||
func TestUNCPathENOTDIR(t *testing.T) {
|
||||
var err error
|
||||
// Instantiate posix object to manage a disk
|
||||
|
||||
+4
-2
@@ -33,7 +33,7 @@ import (
|
||||
)
|
||||
|
||||
func init() {
|
||||
logger.Init(GOPATH)
|
||||
logger.Init(GOPATH, GOROOT)
|
||||
logger.RegisterUIError(fmtError)
|
||||
}
|
||||
|
||||
@@ -82,6 +82,7 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_DRIVES: List of mounted drives or directories delimited by ";".
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";".
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
DOMAIN:
|
||||
MINIO_DOMAIN: To enable virtual-host-style requests, set this value to Minio host domain name.
|
||||
@@ -118,6 +119,7 @@ EXAMPLES:
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
$ export MINIO_CACHE_EXCLUDE="bucket1/*;*.png"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ {{.HelpName}} /home/shared
|
||||
`,
|
||||
}
|
||||
@@ -286,7 +288,7 @@ func serverMain(ctx *cli.Context) {
|
||||
getCert = globalTLSCerts.GetCertificate
|
||||
}
|
||||
|
||||
globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, handler, getCert)
|
||||
globalHTTPServer = xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{handler}, getCert)
|
||||
globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
|
||||
globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
|
||||
go func() {
|
||||
|
||||
+1
-1
@@ -2420,7 +2420,7 @@ func (s *TestSuiteCommon) TestBucketMultipartList(c *check) {
|
||||
// unmarshalling works from a client perspective, specifically
|
||||
// while unmarshalling time.Time type for 'Initiated' field.
|
||||
// time.Time does not honor xml marshaler, it means that we need
|
||||
// to encode/format it before giving it to xml marshalling.
|
||||
// to encode/format it before giving it to xml marshaling.
|
||||
|
||||
// This below check adds client side verification to see if its
|
||||
// truly parseable.
|
||||
|
||||
@@ -39,6 +39,9 @@ func skipContentSha256Cksum(r *http.Request) bool {
|
||||
|
||||
if isRequestPresignedSignatureV4(r) {
|
||||
v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
|
||||
if !ok {
|
||||
v, ok = r.Header["X-Amz-Content-Sha256"]
|
||||
}
|
||||
} else {
|
||||
v, ok = r.Header["X-Amz-Content-Sha256"]
|
||||
}
|
||||
@@ -62,6 +65,9 @@ func getContentSha256Cksum(r *http.Request) string {
|
||||
// will default to 'UNSIGNED-PAYLOAD'.
|
||||
defaultSha256Cksum = unsignedPayload
|
||||
v, ok = r.URL.Query()["X-Amz-Content-Sha256"]
|
||||
if !ok {
|
||||
v, ok = r.Header["X-Amz-Content-Sha256"]
|
||||
}
|
||||
} else {
|
||||
// X-Amz-Content-Sha256, if not set in signed requests, checksum
|
||||
// will default to sha256([]byte("")).
|
||||
|
||||
@@ -76,6 +76,9 @@ var errBitrotHashAlgoInvalid = errors.New("bit-rot hash algorithm is invalid")
|
||||
// errCrossDeviceLink - rename across devices not allowed.
|
||||
var errCrossDeviceLink = errors.New("Rename across devices not allowed, please fix your backend configuration")
|
||||
|
||||
// errMinDiskSize - cannot create volume or files when disk size is less than threshold.
|
||||
var errMinDiskSize = errors.New("The disk size is less than the minimum threshold")
|
||||
|
||||
// hashMisMatchError - represents a bit-rot hash verification failure
|
||||
// error.
|
||||
type hashMismatchError struct {
|
||||
|
||||
@@ -316,9 +316,9 @@ func NewStorageRPCClient(host *xnet.Host, endpointPath string) (*StorageRPCClien
|
||||
// Initialize new storage rpc client.
|
||||
func newStorageRPC(endpoint Endpoint) *StorageRPCClient {
|
||||
host, err := xnet.ParseHost(endpoint.Host)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to parse storage RPC Host", context.Background())
|
||||
rpcClient, err := NewStorageRPCClient(host, endpoint.Path)
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
logger.FatalIf(err, "Unable to initialize storage RPC client", context.Background())
|
||||
rpcClient.connected = rpcClient.Call(storageServiceName+".Connect", &AuthArgs{}, &VoidReply{}) == nil
|
||||
return rpcClient
|
||||
}
|
||||
|
||||
@@ -53,6 +53,12 @@ var (
|
||||
"MINIO_CACHE_EXPIRY: Valid cache expiry duration is in days.",
|
||||
)
|
||||
|
||||
uiErrInvalidCacheMaxUse = newUIErrFn(
|
||||
"Invalid cache max-use value",
|
||||
"Please check the passed value",
|
||||
"MINIO_CACHE_MAXUSE: Valid cache max-use value between 0-100.",
|
||||
)
|
||||
|
||||
uiErrInvalidCredentials = newUIErrFn(
|
||||
"Invalid credentials",
|
||||
"Please provide correct credentials",
|
||||
|
||||
+1
-1
@@ -458,7 +458,7 @@ func getUpdateInfo(timeout time.Duration, mode string) (updateMsg string, sha256
|
||||
|
||||
func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (successMsg string, err error) {
|
||||
if !ok {
|
||||
successMsg = greenColorSprintf("Minio update to version RELEASE.%s cancelled.",
|
||||
successMsg = greenColorSprintf("Minio update to version RELEASE.%s canceled.",
|
||||
latestReleaseTime.Format(minioReleaseTagTimeLayout))
|
||||
return successMsg, nil
|
||||
}
|
||||
|
||||
+3
-6
@@ -458,11 +458,6 @@ type LoginRep struct {
|
||||
func (web *webAPIHandlers) Login(r *http.Request, args *LoginArgs, reply *LoginRep) error {
|
||||
token, err := authenticateWeb(args.Username, args.Password)
|
||||
if err != nil {
|
||||
// Make sure to log errors related to browser login,
|
||||
// for security and auditing reasons.
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("remoteAddr", r.RemoteAddr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
return toJSONError(err)
|
||||
}
|
||||
|
||||
@@ -483,7 +478,9 @@ func (web webAPIHandlers) GenerateAuth(r *http.Request, args *WebGenericArgs, re
|
||||
return toJSONError(errAuthentication)
|
||||
}
|
||||
cred, err := auth.GetNewCredentials()
|
||||
logger.CriticalIf(context.Background(), err)
|
||||
if err != nil {
|
||||
return toJSONError(err)
|
||||
}
|
||||
reply.AccessKey = cred.AccessKey
|
||||
reply.SecretKey = cred.SecretKey
|
||||
reply.UIVersion = browser.UIVersion
|
||||
|
||||
+2
-2
@@ -476,7 +476,7 @@ func (s *xlSets) SetBucketPolicy(ctx context.Context, bucket string, policy *pol
|
||||
|
||||
// GetBucketPolicy will return a policy on a bucket
|
||||
func (s *xlSets) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
|
||||
return GetPolicyConfig(s, bucket)
|
||||
return getPolicyConfig(s, bucket)
|
||||
}
|
||||
|
||||
// DeleteBucketPolicy deletes all policies on bucket
|
||||
@@ -614,7 +614,7 @@ func (s *xlSets) CopyObject(ctx context.Context, srcBucket, srcObject, destBucke
|
||||
}
|
||||
return
|
||||
}
|
||||
// Close writer explicitly signalling we wrote all data.
|
||||
// Close writer explicitly signaling we wrote all data.
|
||||
if gerr := srcInfo.Writer.Close(); gerr != nil {
|
||||
logger.LogIf(ctx, gerr)
|
||||
return
|
||||
|
||||
+1
-1
@@ -284,7 +284,7 @@ func (xl xlObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *
|
||||
|
||||
// GetBucketPolicy will get policy on bucket
|
||||
func (xl xlObjects) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
|
||||
return GetPolicyConfig(xl, bucket)
|
||||
return getPolicyConfig(xl, bucket)
|
||||
}
|
||||
|
||||
// DeleteBucketPolicy deletes all policies on bucket
|
||||
|
||||
@@ -463,7 +463,7 @@ func writeXLMetadata(ctx context.Context, disk StorageAPI, bucket, prefix string
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
// Persist marshalled data.
|
||||
// Persist marshaled data.
|
||||
err = disk.AppendFile(bucket, jsonFile, metadataBytes)
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
|
||||
+13
-4
@@ -277,7 +277,7 @@ func (xl xlObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, ds
|
||||
}
|
||||
return
|
||||
}
|
||||
// Close writer explicitly signalling we wrote all data.
|
||||
// Close writer explicitly signaling we wrote all data.
|
||||
if gerr := srcInfo.Writer.Close(); gerr != nil {
|
||||
logger.LogIf(ctx, gerr)
|
||||
return
|
||||
@@ -375,8 +375,17 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
}
|
||||
|
||||
// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
|
||||
buffer := xl.bp.Get()
|
||||
defer xl.bp.Put(buffer)
|
||||
var buffer []byte
|
||||
switch size := data.Size(); {
|
||||
case size == 0:
|
||||
buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
|
||||
case size < blockSizeV1:
|
||||
// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
|
||||
buffer = make([]byte, size, 2*size)
|
||||
default:
|
||||
buffer = xl.bp.Get()
|
||||
defer xl.bp.Put(buffer)
|
||||
}
|
||||
|
||||
file, err := storage.CreateFile(ctx, data, minioMetaTmpBucket, tmpPartPath, buffer, DefaultBitrotAlgorithm, writeQuorum)
|
||||
if err != nil {
|
||||
@@ -473,7 +482,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
// to indicate where the listing should begin from.
|
||||
//
|
||||
// Implements S3 compatible ListObjectParts API. The resulting
|
||||
// ListPartsInfo structure is marshalled directly into XML and
|
||||
// ListPartsInfo structure is marshaled directly into XML and
|
||||
// replied back to the client.
|
||||
func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) {
|
||||
if err := checkListPartsArgs(ctx, bucket, object, xl); err != nil {
|
||||
|
||||
+12
-3
@@ -142,7 +142,7 @@ func (xl xlObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBuc
|
||||
pipeWriter.CloseWithError(toObjectErr(gerr, srcBucket, srcObject))
|
||||
return
|
||||
}
|
||||
pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
|
||||
pipeWriter.Close() // Close writer explicitly signaling we wrote all data.
|
||||
}()
|
||||
|
||||
hashReader, err := hash.NewReader(pipeReader, length, "", "")
|
||||
@@ -611,8 +611,17 @@ func (xl xlObjects) putObject(ctx context.Context, bucket string, object string,
|
||||
}
|
||||
|
||||
// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
|
||||
buffer := xl.bp.Get()
|
||||
defer xl.bp.Put(buffer)
|
||||
var buffer []byte
|
||||
switch size := data.Size(); {
|
||||
case size == 0:
|
||||
buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
|
||||
case size < blockSizeV1:
|
||||
// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
|
||||
buffer = make([]byte, size, 2*size)
|
||||
default:
|
||||
buffer = xl.bp.Get()
|
||||
defer xl.bp.Put(buffer)
|
||||
}
|
||||
|
||||
// Read data and split into parts - similar to multipart mechanism
|
||||
for partIdx := 1; ; partIdx++ {
|
||||
|
||||
@@ -48,7 +48,7 @@ healthcheck_main () {
|
||||
http_response=$(curl -s -k -o /dev/null -w "%{http_code}" ${scheme}${address}${resource})
|
||||
fi
|
||||
|
||||
# If http_repsonse is 200 - server is up.
|
||||
# If http_response is 200 - server is up.
|
||||
[ "$http_response" = "200" ]
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ Bucket events can be published to the following targets:
|
||||
|
||||
## Prerequisites
|
||||
|
||||
* Install and configure Minio Server from [here](http://docs.minio.io/docs/minio-quickstart-guide).
|
||||
* Install and configure Minio Server from [here](https://docs.minio.io/docs/minio-quickstart-guide).
|
||||
* Install and configure Minio Client from [here](https://docs.minio.io/docs/minio-client-quickstart-guide).
|
||||
|
||||
<a name="AMQP"></a>
|
||||
|
||||
@@ -10,7 +10,7 @@ minio server --config-dir /etc/minio /data
|
||||
```
|
||||
|
||||
### Certificate Directory
|
||||
TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](http://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
|
||||
TLS certificates are stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to Minio server with TLS](https://docs.minio.io/docs/how-to-secure-access-to-minio-server-with-tls).
|
||||
|
||||
Following is the directory structure for Minio server with TLS certificates.
|
||||
|
||||
@@ -115,15 +115,15 @@ By default, parity for objects with standard storage class is set to `N/2`, and
|
||||
|Field|Type|Description|
|
||||
|:---|:---|:---|
|
||||
|``notify``| |Notify enables bucket notification events for lambda computing via the following targets.|
|
||||
|``notify.amqp``| |[Configure to publish Minio events via AMQP target.](http://docs.minio.io/docs/minio-bucket-notification-guide#AMQP)|
|
||||
|``notify.nats``| |[Configure to publish Minio events via NATS target.](http://docs.minio.io/docs/minio-bucket-notification-guide#NATS)|
|
||||
|``notify.elasticsearch``| |[Configure to publish Minio events via Elasticsearch target.](http://docs.minio.io/docs/minio-bucket-notification-guide#Elasticsearch)|
|
||||
|``notify.redis``| |[Configure to publish Minio events via Redis target.](http://docs.minio.io/docs/minio-bucket-notification-guide#Redis)|
|
||||
|``notify.postgresql``| |[Configure to publish Minio events via PostgreSQL target.](http://docs.minio.io/docs/minio-bucket-notification-guide#PostgreSQL)|
|
||||
|``notify.kafka``| |[Configure to publish Minio events via Apache Kafka target.](http://docs.minio.io/docs/minio-bucket-notification-guide#apache-kafka)|
|
||||
|``notify.webhook``| |[Configure to publish Minio events via Webhooks target.](http://docs.minio.io/docs/minio-bucket-notification-guide#webhooks)|
|
||||
|``notify.amqp``| |[Configure to publish Minio events via AMQP target.](https://docs.minio.io/docs/minio-bucket-notification-guide#AMQP)|
|
||||
|``notify.nats``| |[Configure to publish Minio events via NATS target.](https://docs.minio.io/docs/minio-bucket-notification-guide#NATS)|
|
||||
|``notify.elasticsearch``| |[Configure to publish Minio events via Elasticsearch target.](https://docs.minio.io/docs/minio-bucket-notification-guide#Elasticsearch)|
|
||||
|``notify.redis``| |[Configure to publish Minio events via Redis target.](https://docs.minio.io/docs/minio-bucket-notification-guide#Redis)|
|
||||
|``notify.postgresql``| |[Configure to publish Minio events via PostgreSQL target.](https://docs.minio.io/docs/minio-bucket-notification-guide#PostgreSQL)|
|
||||
|``notify.kafka``| |[Configure to publish Minio events via Apache Kafka target.](https://docs.minio.io/docs/minio-bucket-notification-guide#apache-kafka)|
|
||||
|``notify.webhook``| |[Configure to publish Minio events via Webhooks target.](https://docs.minio.io/docs/minio-bucket-notification-guide#webhooks)|
|
||||
|``notify.mysql``| |[Configure to publish Minio events via MySql target.](https://docs.minio.io/docs/minio-bucket-notification-guide#MySQL)|
|
||||
|``notify.mqtt``| |[Configure to publish Minio events via MQTT target.](http://docs.minio.io/docs/minio-bucket-notification-guide#MQTT)|
|
||||
|``notify.mqtt``| |[Configure to publish Minio events via MQTT target.](https://docs.minio.io/docs/minio-bucket-notification-guide#MQTT)|
|
||||
|
||||
## Explore Further
|
||||
* [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)
|
||||
|
||||
@@ -12,6 +12,7 @@ minio server -h
|
||||
MINIO_CACHE_DRIVES: List of mounted cache drives or directories delimited by ";"
|
||||
MINIO_CACHE_EXCLUDE: List of cache exclusion patterns delimited by ";"
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
...
|
||||
...
|
||||
|
||||
@@ -21,6 +22,7 @@ minio server -h
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
|
||||
$ export MINIO_CACHE_EXCLUDE="mybucket/*;*.pdf"
|
||||
$ export MINIO_CACHE_EXPIRY=40
|
||||
$ export MINIO_CACHE_MAXUSE=80
|
||||
$ minio server /home/shared
|
||||
```
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ Disk caching feature here refers to the use of caching disks to store content cl
|
||||
## Get started
|
||||
|
||||
### 1. Prerequisites
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
|
||||
|
||||
### 2. Run Minio with cache
|
||||
Disk caching can be enabled by updating the `cache` config settings for Minio server. Config `cache` settings takes the mounted drive(s) or directory paths, cache expiry duration (in days) and any wildcard patterns to exclude from being cached.
|
||||
@@ -27,6 +27,7 @@ The cache settings may also be set through environment variables. When set, envi
|
||||
export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3"
|
||||
export MINIO_CACHE_EXPIRY=90
|
||||
export MINIO_CACHE_EXCLUDE="*.pdf;mybucket/*"
|
||||
export MINIO_CACHE_MAXUSE=80
|
||||
minio server /export{1...24}
|
||||
```
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ Erasure code protects data from multiple drives failure, unlike RAID or replicat
|
||||
|
||||
## What is Bit Rot protection?
|
||||
|
||||
Bit Rot, also known as data rot or silent data corruption is a data loss issue faced by disk drives today. Data on the drive may silently get corrupted without signalling an error has occurred, making bit rot more dangerous than a permanent hard drive failure.
|
||||
Bit Rot, also known as data rot or silent data corruption is a data loss issue faced by disk drives today. Data on the drive may silently get corrupted without signaling an error has occurred, making bit rot more dangerous than a permanent hard drive failure.
|
||||
|
||||
Minio's erasure coded backend uses high speed [HighwayHash](https://blog.minio.io/highwayhash-fast-hashing-at-over-10-gb-s-per-core-in-golang-fee938b5218a) checksums to protect against Bit Rot.
|
||||
|
||||
|
||||
@@ -11,8 +11,8 @@ saving the object in specific number of data and parity disks.
|
||||
|
||||
## Storage usage
|
||||
|
||||
The selection of varying data and parity drives has a direct impact on the drive space usage. With storage class, you can optimise for high
|
||||
redundancy or better drive space utilisation.
|
||||
The selection of varying data and parity drives has a direct impact on the drive space usage. With storage class, you can optimize for high
|
||||
redundancy or better drive space utilization.
|
||||
|
||||
To get an idea of how various combinations of data and parity drives affect the storage usage, let’s take an example of a 100 MiB file stored
|
||||
on 16 drive Minio deployment. If you use eight data and eight parity drives, the file space usage will be approximately twice, i.e. 100 MiB
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
}
|
||||
debug
|
||||
prometheus
|
||||
cache 160 churchofminio.com
|
||||
cache 160 mydomain.com
|
||||
loadbalance
|
||||
proxy . /etc/resolv.conf
|
||||
}
|
||||
@@ -1,15 +1,20 @@
|
||||
# Federation
|
||||
# Federation Quickstart Guide [](https://slack.minio.io)
|
||||
This document explains how to configure Minio with `Bucket lookup from DNS` style federation.
|
||||
|
||||
## Dependencies
|
||||
## Get started
|
||||
|
||||
### 1. Prerequisites
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
|
||||
|
||||
### 2. Run Minio in federated mode
|
||||
Bucket lookup from DNS federation requires two dependencies
|
||||
|
||||
- etcd (for config, bucket SRV records)
|
||||
- coredns (for DNS management based on populated bucket SRV records)
|
||||
- coredns (for DNS management based on populated bucket SRV records, optional)
|
||||
|
||||
## Architecture
|
||||
|
||||

|
||||

|
||||
|
||||
### Environment variables
|
||||
|
||||
@@ -36,7 +41,7 @@ a bucket `bucket1` created on current Minio instance will be accessible as `buck
|
||||
|
||||
- This field is mandatory for standalone and erasure code Minio server deployments, to enable federated mode.
|
||||
- This field is optional for distributed deployments. If you don't set this field in a federated setup, we use the IP addresses of
|
||||
hosts passed to the Minio server startup and use that to make DNS entries.
|
||||
hosts passed to the Minio server startup and use them for DNS entries.
|
||||
|
||||
### Run Multiple Clusters
|
||||
|
||||
@@ -67,3 +72,13 @@ points to the public IP address where each cluster might be accessible, this is
|
||||
NOTE: `mybucket` only exists on one cluster either `cluster1` or `cluster2` this is random and
|
||||
is decided by how `domain.com` gets resolved, if there is a round-robin DNS on `domain.com` then
|
||||
it is randomized which cluster might provision the bucket.
|
||||
|
||||
### 3. Test your setup
|
||||
To test this setup, access the Minio server via browser or [`mc`](https://docs.minio.io/docs/minio-client-quickstart-guide). You’ll see the uploaded files are accessible from the all the Minio endpoints.
|
||||
|
||||
# Explore Further
|
||||
- [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
|
||||
- [Use `aws-cli` with Minio Server](https://docs.minio.io/docs/aws-cli-with-minio)
|
||||
- [Use `s3cmd` with Minio Server](https://docs.minio.io/docs/s3cmd-with-minio)
|
||||
- [Use `minio-go` SDK with Minio Server](https://docs.minio.io/docs/golang-client-quickstart-guide)
|
||||
- [The Minio documentation website](https://docs.minio.io)
|
||||
|
||||
@@ -8,7 +8,7 @@ We call it large-bucket as it allows a single Minio bucket to expand over multip
|
||||
If you're aware of distributed Minio setup, the installation and configuration remains the same. Only new addition to the input syntax is `...` convention to abbreviate the drive arguments. Remote drives in a distributed setup are encoded as HTTP(s) URIs which can be similarly abbreviated as well.
|
||||
|
||||
### 1. Prerequisites
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
|
||||
|
||||
### 2. Run Minio on many drives
|
||||
We'll see examples on how to do this in the following sections.
|
||||
|
||||
@@ -36,19 +36,19 @@ We found the following APIs to be redundant or less useful outside of AWS S3. If
|
||||
|
||||
#### List of Amazon S3 Bucket API's not supported on Minio
|
||||
|
||||
- BucketACL (Use [bucket policies](http://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
|
||||
- BucketACL (Use [bucket policies](https://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
|
||||
- BucketCORS (CORS enabled by default on all buckets for all HTTP verbs)
|
||||
- BucketLifecycle (Not required for Minio erasure coded backend)
|
||||
- BucketReplication (Use [`mc mirror`](http://docs.minio.io/docs/minio-client-complete-guide#mirror) instead)
|
||||
- BucketReplication (Use [`mc mirror`](https://docs.minio.io/docs/minio-client-complete-guide#mirror) instead)
|
||||
- BucketVersions, BucketVersioning (Use [`s3git`](https://github.com/s3git/s3git))
|
||||
- BucketWebsite (Use [`caddy`](https://github.com/mholt/caddy) or [`nginx`](https://www.nginx.com/resources/wiki/))
|
||||
- BucketAnalytics, BucketMetrics, BucketLogging (Use [bucket notification](http://docs.minio.io/docs/minio-client-complete-guide#events) APIs)
|
||||
- BucketAnalytics, BucketMetrics, BucketLogging (Use [bucket notification](https://docs.minio.io/docs/minio-client-complete-guide#events) APIs)
|
||||
- BucketRequestPayment
|
||||
- BucketTagging
|
||||
|
||||
#### List of Amazon S3 Object API's not supported on Minio
|
||||
|
||||
- ObjectACL (Use [bucket policies](http://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
|
||||
- ObjectACL (Use [bucket policies](https://docs.minio.io/docs/minio-client-complete-guide#policy) instead)
|
||||
- ObjectTorrent
|
||||
- ObjectVersions
|
||||
|
||||
|
||||
@@ -4,10 +4,10 @@ Minio is a cloud-native application designed to scale in a sustainable manner in
|
||||
|
||||
| Orchestration platforms|
|
||||
|:---|
|
||||
| [`Docker Swarm`](http://docs.minio.io/docs/deploy-minio-on-docker-swarm) |
|
||||
| [`Docker Compose`](http://docs.minio.io/docs/deploy-minio-on-docker-compose) |
|
||||
| [`Kubernetes`](http://docs.minio.io/docs/deploy-minio-on-kubernetes) |
|
||||
| [`DC/OS`](http://docs.minio.io/docs/deploy-minio-on-dc-os) |
|
||||
| [`Docker Swarm`](https://docs.minio.io/docs/deploy-minio-on-docker-swarm) |
|
||||
| [`Docker Compose`](https://docs.minio.io/docs/deploy-minio-on-docker-compose) |
|
||||
| [`Kubernetes`](https://docs.minio.io/docs/deploy-minio-on-kubernetes) |
|
||||
| [`DC/OS`](https://docs.minio.io/docs/deploy-minio-on-dc-os) |
|
||||
|
||||
## Why is Minio cloud-native?
|
||||
The term cloud-native revolves around the idea of applications deployed as micro services, that scale well. It is not about just retrofitting monolithic applications onto modern container based compute environment. A cloud-native application is portable and resilient by design, and can scale horizontally by simply replicating. Modern orchestration platforms like Swarm, Kubernetes and DC/OS make replicating and managing containers in huge clusters easier than ever.
|
||||
|
||||
@@ -5,7 +5,7 @@ version: '2'
|
||||
# 9001 through 9004.
|
||||
services:
|
||||
minio1:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- data1:/data
|
||||
ports:
|
||||
@@ -15,7 +15,7 @@ services:
|
||||
MINIO_SECRET_KEY: minio123
|
||||
command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data
|
||||
minio2:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- data2:/data
|
||||
ports:
|
||||
@@ -25,7 +25,7 @@ services:
|
||||
MINIO_SECRET_KEY: minio123
|
||||
command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data
|
||||
minio3:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- data3:/data
|
||||
ports:
|
||||
@@ -35,7 +35,7 @@ services:
|
||||
MINIO_SECRET_KEY: minio123
|
||||
command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data
|
||||
minio4:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- data4:/data
|
||||
ports:
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.1'
|
||||
|
||||
services:
|
||||
minio1:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio1-data:/export
|
||||
ports:
|
||||
@@ -20,7 +20,7 @@ services:
|
||||
- access_key
|
||||
|
||||
minio2:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio2-data:/export
|
||||
ports:
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
- access_key
|
||||
|
||||
minio3:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio3-data:/export
|
||||
ports:
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
- access_key
|
||||
|
||||
minio4:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio4-data:/export
|
||||
ports:
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3'
|
||||
|
||||
services:
|
||||
minio1:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio1-data:/export
|
||||
ports:
|
||||
@@ -20,7 +20,7 @@ services:
|
||||
command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
|
||||
|
||||
minio2:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio2-data:/export
|
||||
ports:
|
||||
@@ -38,7 +38,7 @@ services:
|
||||
command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
|
||||
|
||||
minio3:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio3-data:/export
|
||||
ports:
|
||||
@@ -56,7 +56,7 @@ services:
|
||||
command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
|
||||
|
||||
minio4:
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
volumes:
|
||||
- minio4-data:/export
|
||||
ports:
|
||||
|
||||
@@ -124,7 +124,7 @@ spec:
|
||||
- name: data
|
||||
mountPath: "/data"
|
||||
# Pulls the lastest Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
args:
|
||||
- server
|
||||
- /data
|
||||
@@ -303,7 +303,7 @@ spec:
|
||||
value: "minio"
|
||||
- name: MINIO_SECRET_KEY
|
||||
value: "minio123"
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
args:
|
||||
- server
|
||||
- http://minio-0.minio.default.svc.cluster.local/data
|
||||
@@ -514,7 +514,7 @@ spec:
|
||||
containers:
|
||||
- name: minio
|
||||
# Pulls the default Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
args:
|
||||
- gateway
|
||||
- gcs
|
||||
|
||||
@@ -21,7 +21,7 @@ spec:
|
||||
value: "minio"
|
||||
- name: MINIO_SECRET_KEY
|
||||
value: "minio123"
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
args:
|
||||
- server
|
||||
- http://minio-0.minio.default.svc.cluster.local/data
|
||||
|
||||
@@ -21,7 +21,7 @@ spec:
|
||||
containers:
|
||||
- name: minio
|
||||
# Pulls the default Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
args:
|
||||
- gateway
|
||||
- gcs
|
||||
|
||||
@@ -29,7 +29,7 @@ spec:
|
||||
- name: data
|
||||
mountPath: "/data"
|
||||
# Pulls the lastest Minio image from Docker Hub
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
args:
|
||||
- server
|
||||
- /data
|
||||
|
||||
@@ -30,7 +30,7 @@ Above command deploys Minio on the Kubernetes cluster in the default configurati
|
||||
| Parameter | Description | Default |
|
||||
|----------------------------|-------------------------------------|---------------------------------------------------------|
|
||||
| `image` | Minio image name | `minio/minio` |
|
||||
| `imageTag` | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-06-08T03-49-38Z`|
|
||||
| `imageTag` | Minio image tag. Possible values listed [here](https://hub.docker.com/r/minio/minio/tags/).| `RELEASE.2018-06-22T23-48-46Z`|
|
||||
| `imagePullPolicy` | Image pull policy | `Always` |
|
||||
| `mode` | Minio server mode (`standalone`, `shared` or `distributed`)| `standalone` |
|
||||
| `numberOfNodes` | Number of nodes (applicable only for Minio distributed mode). Should be 4 <= x <= 16 | `4` |
|
||||
|
||||
@@ -54,7 +54,7 @@ spec:
|
||||
value: "minio"
|
||||
- name: MINIO_SECRET_KEY
|
||||
value: "minio123"
|
||||
image: minio/minio:RELEASE.2018-06-08T03-49-38Z
|
||||
image: minio/minio:RELEASE.2018-06-22T23-48-46Z
|
||||
imagePullPolicy: IfNotPresent
|
||||
args: ["server", "http://minio-0.minio.default.svc.cluster.local/data", "http://minio-1.minio.default.svc.cluster.local/data", "http://minio-2.minio.default.svc.cluster.local/data", "http://minio-3.minio.default.svc.cluster.local/data"]
|
||||
ports:
|
||||
|
||||
@@ -18,7 +18,7 @@ If you're aware of stand-alone Minio set up, the installation and running remain
|
||||
|
||||
## 1. Prerequisites
|
||||
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio).
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
|
||||
|
||||
## 2. Run Minio on Shared Backend
|
||||
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
|
||||
## 前提条件
|
||||
|
||||
* 从[这里](http://docs.minio.io/docs/zh_CN/minio-quickstart-guide)下载并安装Minio Server。
|
||||
* 从[这里](https://docs.minio.io/docs/zh_CN/minio-client-quickstart-guide)下载并安装Minio Client。
|
||||
* 从[这里](https://docs.minio.io/cn/minio-quickstart-guide)下载并安装Minio Server。
|
||||
* 从[这里](https://docs.minio.io/cn/minio-client-quickstart-guide)下载并安装Minio Client。
|
||||
|
||||
<a name="AMQP"></a>
|
||||
## 使用AMQP发布Minio事件
|
||||
@@ -960,4 +960,4 @@ mc ls myminio/images-thumbnail
|
||||
[2017-02-08 11:39:40 IST] 992B images-thumbnail.jpg
|
||||
```
|
||||
|
||||
*注意* 如果你用的是 [distributed Minio](https://docs.minio.io/docs/zh_CN/distributed-minio-quickstart-guide),请修改所有节点的 ``~/.minio/config.json``。
|
||||
*注意* 如果你用的是 [distributed Minio](https://docs.minio.io/cn/distributed-minio-quickstart-guide),请修改所有节点的 ``~/.minio/config.json``。
|
||||
|
||||
@@ -10,7 +10,7 @@ minio server --config-dir /etc/minio /data
|
||||
```
|
||||
|
||||
### 证书目录
|
||||
TLS证书存在``${HOME}/.minio/certs``目录下,你需要将证书放在该目录下来启用`HTTPS` 。如果你是一个乐学上进的好青年,这里有一本免费的秘籍传授一你: [如何使用TLS安全的访问minio](http://docs.minio.io/docs/zh_CN/how-to-secure-access-to-minio-server-with-tls).
|
||||
TLS证书存在``${HOME}/.minio/certs``目录下,你需要将证书放在该目录下来启用`HTTPS` 。如果你是一个乐学上进的好青年,这里有一本免费的秘籍传授一你: [如何使用TLS安全的访问minio](https://docs.minio.io/cn/how-to-secure-access-to-minio-server-with-tls).
|
||||
|
||||
以下是一个带来TLS证书的Minio server的目录结构。
|
||||
|
||||
@@ -73,14 +73,14 @@ minio server /data
|
||||
|参数|类型|描述|
|
||||
|:---|:---|:---|
|
||||
|``notify``| |通知通过以下方式开启存储桶事件通知,用于lambda计算|
|
||||
|``notify.amqp``| |[通过AMQP发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#AMQP)|
|
||||
|``notify.mqtt``| |[通过MQTT发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#MQTT)|
|
||||
|``notify.elasticsearch``| |[通过Elasticsearch发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#Elasticsearch)|
|
||||
|``notify.redis``| |[通过Redis发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#Redis)|
|
||||
|``notify.nats``| |[通过NATS发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#NATS)|
|
||||
|``notify.postgresql``| |[通过PostgreSQL发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#PostgreSQL)|
|
||||
|``notify.kafka``| |[通过Apache Kafka发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#apache-kafka)|
|
||||
|``notify.webhook``| |[通过Webhooks发布Minio事件](http://docs.minio.io/docs/zh_CN/minio-bucket-notification-guide#webhooks)|
|
||||
|``notify.amqp``| |[通过AMQP发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#AMQP)|
|
||||
|``notify.mqtt``| |[通过MQTT发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#MQTT)|
|
||||
|``notify.elasticsearch``| |[通过Elasticsearch发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#Elasticsearch)|
|
||||
|``notify.redis``| |[通过Redis发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#Redis)|
|
||||
|``notify.nats``| |[通过NATS发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#NATS)|
|
||||
|``notify.postgresql``| |[通过PostgreSQL发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#PostgreSQL)|
|
||||
|``notify.kafka``| |[通过Apache Kafka发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#apache-kafka)|
|
||||
|``notify.webhook``| |[通过Webhooks发布Minio事件](https://docs.minio.io/cn/minio-bucket-notification-guide#webhooks)|
|
||||
|
||||
## 了解更多
|
||||
* [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
## 开始
|
||||
|
||||
### 1. 前期条件
|
||||
安装Minio - [Minio快速入门](https://docs.minio.io/docs/minio)。
|
||||
安装Minio - [Minio快速入门](https://docs.minio.io/docs/minio-quickstart-guide)。
|
||||
|
||||
### 2. 运行Minio缓存
|
||||
磁盘缓存可以通过修改Minio服务的`cache`配置来进行开启。配置`cache`设置需要指定磁盘路径、缓存过期时间(以天为单位)以及使用统配符方式指定的不需要进行缓存的对象。
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
|
||||
### 数据保护
|
||||
|
||||
分布式Minio采用 [erasure code](https://docs.minio.io/docs/zh_CN/minio-erasure-code-quickstart-guide)来防范多个节点宕机和[位衰减`bit rot`](https://github.com/minio/minio/blob/master/docs/zh_CN/erasure/README.md#what-is-bit-rot-protection)。
|
||||
分布式Minio采用 [erasure code](https://docs.minio.io/cn/minio-erasure-code-quickstart-guide)来防范多个节点宕机和[位衰减`bit rot`](https://github.com/minio/minio/blob/master/cn/erasure/README.md#what-is-bit-rot-protection)。
|
||||
分布式Minio至少需要4个节点,使用分布式Minio自动引入了纠删码功能。
|
||||
|
||||
### 高可用
|
||||
@@ -33,7 +33,7 @@ Minio在分布式和单机模式下,所有读写操作都严格遵守**read-af
|
||||
|
||||
## 1. 前提条件
|
||||
|
||||
安装Minio - [Minio快速入门](https://docs.minio.io/docs/zh_CN/minio-quickstart-guide).
|
||||
安装Minio - [Minio快速入门](https://docs.minio.io/cn/minio-quickstart-guide).
|
||||
|
||||
## 2. 运行分布式Minio
|
||||
|
||||
@@ -108,12 +108,12 @@ minio.exe server http://192.168.1.11/C:/data1 http://192.168.1.11/C:/data2 ^
|
||||
|
||||
## 3. 验证
|
||||
|
||||
验证是否部署成功,使用浏览器访问Minio服务或者使用 [`mc`](https://docs.minio.io/docs/zh_CN/minio-client-quickstart-guide)。多个节点的存储容量和就是分布式Minio的存储容量。
|
||||
验证是否部署成功,使用浏览器访问Minio服务或者使用 [`mc`](https://docs.minio.io/cn/minio-client-quickstart-guide)。多个节点的存储容量和就是分布式Minio的存储容量。
|
||||
|
||||
## 了解更多
|
||||
- [Minio纠删码快速入门](https://docs.minio.io/docs/zh_CN/minio-erasure-code-quickstart-guide)
|
||||
- [使用 `mc`](https://docs.minio.io/docs/zh_CN/minio-client-quickstart-guide)
|
||||
- [使用 `aws-cli`](https://docs.minio.io/docs/zh_CN/aws-cli-with-minio)
|
||||
- [使用 `s3cmd](https://docs.minio.io/docs/zh_CN/s3cmd-with-minio)
|
||||
- [使用 `minio-go` SDK ](https://docs.minio.io/docs/zh_CN/golang-client-quickstart-guide)
|
||||
- [Minio纠删码快速入门](https://docs.minio.io/cn/minio-erasure-code-quickstart-guide)
|
||||
- [使用 `mc`](https://docs.minio.io/cn/minio-client-quickstart-guide)
|
||||
- [使用 `aws-cli`](https://docs.minio.io/cn/aws-cli-with-minio)
|
||||
- [使用 `s3cmd](https://docs.minio.io/cn/s3cmd-with-minio)
|
||||
- [使用 `minio-go` SDK ](https://docs.minio.io/cn/golang-client-quickstart-guide)
|
||||
- [minio官方文档](https://docs.minio.io)
|
||||
|
||||
@@ -29,7 +29,7 @@ docker run -p 9000:9000 --name minio1 \
|
||||
```
|
||||
|
||||
## 在Docker中运行Minio分布式模式
|
||||
分布式Minio可以通过 [Docker Compose](https://docs.minio.io/docs/zh_CN/deploy-minio-on-docker-compose) 或者 [Swarm mode](https://docs.minio.io/docs/zh_CN/deploy-minio-on-docker-swarm)进行部署。这两者之间的主要区别是Docker Compose创建了单个主机,多容器部署,而Swarm模式创建了一个多主机,多容器部署。
|
||||
分布式Minio可以通过 [Docker Compose](https://docs.minio.io/cn/deploy-minio-on-docker-compose) 或者 [Swarm mode](https://docs.minio.io/cn/deploy-minio-on-docker-swarm)进行部署。这两者之间的主要区别是Docker Compose创建了单个主机,多容器部署,而Swarm模式创建了一个多主机,多容器部署。
|
||||
|
||||
这意味着Docker Compose可以让你快速的在你的机器上快速使用分布式Minio-非常适合开发,测试环境;而Swarm模式提供了更健壮,生产级别的部署。
|
||||
|
||||
@@ -110,7 +110,7 @@ docker stats <container_id>
|
||||
|
||||
## 了解更多
|
||||
|
||||
* [在Docker Compose上部署Minio](https://docs.minio.io/docs/zh_CN/deploy-minio-on-docker-compose)
|
||||
* [在Docker Swarm上部署Minio](https://docs.minio.io/docs/zh_CN/deploy-minio-on-docker-swarm)
|
||||
* [分布式Minio快速入门](https://docs.minio.io/docs/zh_CN/distributed-minio-quickstart-guide)
|
||||
* [Minio纠删码模式快速入门](https://docs.minio.io/docs/zh_CN/minio-erasure-code-quickstart-guide)
|
||||
* [在Docker Compose上部署Minio](https://docs.minio.io/cn/deploy-minio-on-docker-compose)
|
||||
* [在Docker Swarm上部署Minio](https://docs.minio.io/cn/deploy-minio-on-docker-swarm)
|
||||
* [分布式Minio快速入门](https://docs.minio.io/cn/distributed-minio-quickstart-guide)
|
||||
* [Minio纠删码模式快速入门](https://docs.minio.io/cn/minio-erasure-code-quickstart-guide)
|
||||
|
||||
@@ -20,7 +20,7 @@ Minio使用纠删码`erasure code`和校验和`checksum`来保护数据免受硬
|
||||
|
||||
### 1. 前提条件:
|
||||
|
||||
安装Minio- [Minio快速入门](https://docs.minio.io/docs/zh_CN/minio-quickstart-guide)
|
||||
安装Minio- [Minio快速入门](https://docs.minio.io/cn/minio-quickstart-guide)
|
||||
|
||||
### 2. 以纠删码模式运行Minio
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user