mirror of
https://github.com/pgsty/minio.git
synced 2026-11-16 20:39:35 +03:00
Compare commits
89 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| d1a6c32d80 | |||
| bbd05a8f1c | |||
| 4393afb7e2 | |||
| 1a53734477 | |||
| e95c0bb913 | |||
| fce556b8a0 | |||
| b97f99766f | |||
| f7b6f7b22f | |||
| 2a0a62b78d | |||
| 67f66c40c1 | |||
| 5a2bdf6959 | |||
| 51e78a3e20 | |||
| 9eb52ec7c7 | |||
| 0827a2747b | |||
| e7a724de0d | |||
| d57d57ddf5 | |||
| a4d6195244 | |||
| 7d3eaf79ff | |||
| b63c37b28e | |||
| 8d59f35523 | |||
| f460eceb6d | |||
| d10679866c | |||
| a79a7e570c | |||
| 16ecaac4fc | |||
| ca6b4773ed | |||
| 7e7ae29d89 | |||
| 7d18f00116 | |||
| 719f8c258a | |||
| dcdb07433a | |||
| 26e9f78a86 | |||
| 34a1b58a75 | |||
| 32c6b62932 | |||
| 8d584bd819 | |||
| 031767791d | |||
| 3b917067d9 | |||
| 203ac8edaa | |||
| 8bbfb1b714 | |||
| b4ddccc2f7 | |||
| 6400f506da | |||
| 7195ac7f14 | |||
| 03df692ae2 | |||
| a011fe8450 | |||
| 95d97c2d6d | |||
| bc8b936d4b | |||
| d23ded0d83 | |||
| db3fed2279 | |||
| 866dffcd62 | |||
| 5eb210dd2e | |||
| 758d5458f0 | |||
| 8a40da3fd0 | |||
| 1d8a8c63db | |||
| 65a817fe8c | |||
| 1256b0b818 | |||
| 7e05b826fa | |||
| d82a1da511 | |||
| f598f4fd1b | |||
| e65d8fdc7d | |||
| b919462610 | |||
| 8287ab091c | |||
| f25bec6bf1 | |||
| 75865efb0e | |||
| 3d2d63f71e | |||
| 53e133e844 | |||
| b2cbade477 | |||
| 0c0d1e4150 | |||
| 3d0dced23c | |||
| ad53c5d859 | |||
| 45463b1d6b | |||
| d5895d3243 | |||
| db1edfe487 | |||
| 9c16f73334 | |||
| f50bec2987 | |||
| e2bff5cdc0 | |||
| 02a5f1e96a | |||
| 4deefa3695 | |||
| d28b3d8801 | |||
| 099b5293a9 | |||
| 4a0a491ca1 | |||
| db6b6e9518 | |||
| d8a11c8f4b | |||
| 0b546ddfd4 | |||
| d1712a46a7 | |||
| c46c2a6dd6 | |||
| 88938340b3 | |||
| 13a7033505 | |||
| 670e3e0c8b | |||
| 89d528a4ed | |||
| 53f3d2fd65 | |||
| a4f26aec00 |
@@ -24,5 +24,6 @@
|
||||
- [ ] My code follows the code style of this project.
|
||||
- [ ] My change requires a change to the documentation.
|
||||
- [ ] I have updated the documentation accordingly.
|
||||
- [ ] I have added tests to cover my changes.
|
||||
- [ ] I have added unit tests to cover my changes.
|
||||
- [ ] I have added/updated functional tests in [mint](https://github.com/minio/mint). (If yes, add `mint` PR # here: )
|
||||
- [ ] All new and existing tests passed.
|
||||
+1
-2
@@ -13,7 +13,6 @@ os:
|
||||
|
||||
env:
|
||||
- ARCH=x86_64
|
||||
- ARCH=i686
|
||||
|
||||
script:
|
||||
## Run all the tests
|
||||
@@ -27,4 +26,4 @@ after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
go:
|
||||
- 1.8.x
|
||||
- 1.9.1
|
||||
|
||||
+8
-7
@@ -1,4 +1,4 @@
|
||||
FROM alpine:3.6
|
||||
FROM golang:1.9.1-alpine3.6
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
@@ -8,9 +8,11 @@ ENV CGO_ENABLED 0
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
|
||||
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps git go musl-dev && \
|
||||
apk add --no-cache ca-certificates curl && \
|
||||
apk add --no-cache --virtual .build-deps git && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
@@ -19,12 +21,11 @@ RUN \
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s \
|
||||
CMD /usr/bin/healthcheck.sh
|
||||
|
||||
CMD ["minio"]
|
||||
|
||||
@@ -0,0 +1,32 @@
|
||||
FROM golang:1.9.1-alpine3.6
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
|
||||
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
|
||||
|
||||
COPY . /go/src/github.com/minio/minio
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates curl && \
|
||||
apk add --no-cache --virtual .build-deps git && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
|
||||
rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s \
|
||||
CMD /usr/bin/healthcheck.sh
|
||||
|
||||
CMD ["minio"]
|
||||
+1
-1
@@ -2,7 +2,7 @@ FROM alpine:3.6
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh buildscripts/healthcheck.sh /usr/bin/
|
||||
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
|
||||
@@ -19,7 +19,7 @@ getdeps: checks
|
||||
@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
|
||||
@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
|
||||
|
||||
verifiers: getdeps vet fmt lint cyclo spelling
|
||||
verifiers: getdeps vet fmt lint cyclo deadcode spelling
|
||||
|
||||
vet:
|
||||
@echo "Running $@"
|
||||
@@ -46,7 +46,8 @@ cyclo:
|
||||
@${GOPATH}/bin/gocyclo -over 100 pkg
|
||||
|
||||
deadcode:
|
||||
@${GOPATH}/bin/deadcode
|
||||
@echo "Running $@"
|
||||
@${GOPATH}/bin/deadcode -test $(shell go list ./...) || true
|
||||
|
||||
spelling:
|
||||
@${GOPATH}/bin/misspell -error `find cmd/`
|
||||
@@ -69,7 +70,7 @@ coverage: build
|
||||
|
||||
# Builds minio locally.
|
||||
build:
|
||||
@echo "Building minio to $(PWD)/minio ..."
|
||||
@echo "Building minio binary: $(PWD)/minio"
|
||||
@CGO_ENABLED=0 go build --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
|
||||
|
||||
pkg-add:
|
||||
@@ -89,15 +90,9 @@ pkg-list:
|
||||
|
||||
# Builds minio and installs it to $GOPATH/bin.
|
||||
install: build
|
||||
@echo "Installing minio at $(GOPATH)/bin/minio ..."
|
||||
@echo "Installing minio binary: $(GOPATH)/bin/minio"
|
||||
@cp $(PWD)/minio $(GOPATH)/bin/minio
|
||||
@echo "Check 'minio -h' for help."
|
||||
|
||||
release: verifiers
|
||||
@MINIO_RELEASE=RELEASE ./buildscripts/build.sh
|
||||
|
||||
experimental: verifiers
|
||||
@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
|
||||
@echo "\nInstallation successful. To learn more, try \"minio --help\"."
|
||||
|
||||
clean:
|
||||
@echo "Cleaning up all the generated files"
|
||||
|
||||
+10
-8
@@ -12,7 +12,7 @@ clone_folder: c:\gopath\src\github.com\minio\minio
|
||||
# Environment variables
|
||||
environment:
|
||||
GOPATH: c:\gopath
|
||||
GOROOT: c:\go18
|
||||
GOROOT: c:\go
|
||||
|
||||
# scripts that run after cloning repository
|
||||
install:
|
||||
@@ -40,17 +40,19 @@ test_script:
|
||||
- mkdir build\coverage
|
||||
- go test -v -timeout 17m -race github.com/minio/minio/cmd...
|
||||
- go test -v -race github.com/minio/minio/pkg...
|
||||
- go test -v -timeout 17m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
|
||||
# FIXME(aead): enable codecov after issue https://github.com/golang/go/issues/18468 is solved.
|
||||
# - go test -v -timeout 17m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
|
||||
- ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
|
||||
|
||||
after_test:
|
||||
- go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
|
||||
- ps: Push-AppveyorArtifact build\coverage\coverage.txt
|
||||
- ps: Push-AppveyorArtifact build\coverage\coverage.html
|
||||
# FIXME(aead): enable codecov after issue https://github.com/golang/go/issues/18468 is solved.
|
||||
# - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
|
||||
# - ps: Push-AppveyorArtifact build\coverage\coverage.txt
|
||||
# - ps: Push-AppveyorArtifact build\coverage\coverage.html
|
||||
# Upload coverage report.
|
||||
- "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
|
||||
- pip install codecov
|
||||
- codecov -X gcov -f "build\coverage\coverage.txt"
|
||||
# - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
|
||||
# - pip install codecov
|
||||
# - codecov -X gcov -f "build\coverage\coverage.txt"
|
||||
|
||||
# to disable deployment
|
||||
deploy: off
|
||||
|
||||
@@ -5,13 +5,13 @@
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Minio Browser</title>
|
||||
<link rel="stylesheet" href="/minio/loader.css" type="text/css">
|
||||
<link rel="stylesheet" href="loader.css" type="text/css">
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div class="page-load">
|
||||
<div class="pl-inner">
|
||||
<img src="/minio/logo.svg" alt="">
|
||||
<img src="logo.svg" alt="">
|
||||
</div>
|
||||
</div>
|
||||
<div id="root"></div>
|
||||
@@ -27,19 +27,19 @@
|
||||
<ul>
|
||||
<li>
|
||||
<a href="http://www.google.com/chrome/">
|
||||
<img src="/minio/chrome.png" alt="">
|
||||
<img src="chrome.png" alt="">
|
||||
<div>Chrome</div>
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="https://www.mozilla.org/en-US/firefox/new/">
|
||||
<img src="/minio/firefox.png" alt="">
|
||||
<img src="firefox.png" alt="">
|
||||
<div>Firefox</div>
|
||||
</a>
|
||||
</li>
|
||||
<li>
|
||||
<a href="https://www.apple.com/safari/">
|
||||
<img src="/minio/safari.png" alt="">
|
||||
<img src="safari.png" alt="">
|
||||
<div>Safari</div>
|
||||
</a>
|
||||
</li>
|
||||
@@ -51,6 +51,6 @@
|
||||
<![endif]-->
|
||||
|
||||
<script>currentUiVersion = 'MINIO_UI_VERSION'</script>
|
||||
<script src="/minio/index_bundle.js"></script>
|
||||
<script src="index_bundle.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -81,7 +81,7 @@ ReactDOM.render((
|
||||
<Provider store={ store } web={ web }>
|
||||
<Router history={ browserHistory }>
|
||||
<Route path='/' component={ App }>
|
||||
<Route path='minio' component={ App }>
|
||||
<Route path={ minioBrowserPrefix } component={ App }>
|
||||
<IndexRoute component={ Browse } onEnter={ authNeeded } />
|
||||
<Route path='login' component={ Login } onEnter={ authNotNeeded } />
|
||||
<Route path=':bucket' component={ Browse } onEnter={ authNeeded } />
|
||||
|
||||
@@ -465,7 +465,7 @@ export const uploadFile = (file, xhr) => {
|
||||
return (dispatch, getState) => {
|
||||
const {currentBucket, currentPath} = getState()
|
||||
const objectName = `${currentPath}${file.name}`
|
||||
const uploadUrl = `${window.location.origin}/minio/upload/${currentBucket}/${objectName}`
|
||||
const uploadUrl = `${window.location.origin}${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
|
||||
// The slug is a unique identifer for the file upload.
|
||||
const slug = `${currentBucket}-${currentPath}-${file.name}`
|
||||
|
||||
@@ -482,7 +482,7 @@ export const uploadFile = (file, xhr) => {
|
||||
}))
|
||||
|
||||
xhr.onload = function(event) {
|
||||
if (xhr.status == 401 || xhr.status == 403 || xhr.status == 500) {
|
||||
if (xhr.status == 401 || xhr.status == 403) {
|
||||
setShowAbortModal(false)
|
||||
dispatch(stopUpload({
|
||||
slug
|
||||
@@ -492,6 +492,16 @@ export const uploadFile = (file, xhr) => {
|
||||
message: 'Unauthorized request.'
|
||||
}))
|
||||
}
|
||||
if (xhr.status == 500) {
|
||||
setShowAbortModal(false)
|
||||
dispatch(stopUpload({
|
||||
slug
|
||||
}))
|
||||
dispatch(showAlert({
|
||||
type: 'danger',
|
||||
message: xhr.responseText
|
||||
}))
|
||||
}
|
||||
if (xhr.status == 200) {
|
||||
setShowAbortModal(false)
|
||||
dispatch(stopUpload({
|
||||
|
||||
@@ -150,16 +150,21 @@ export default class Browse extends React.Component {
|
||||
if (prefix === currentPath) return
|
||||
browserHistory.push(utils.pathJoin(currentBucket, encPrefix))
|
||||
} else {
|
||||
// Download the selected file.
|
||||
web.CreateURLToken()
|
||||
.then(res => {
|
||||
let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=${res.token}`
|
||||
window.location = url
|
||||
})
|
||||
.catch(err => dispatch(actions.showAlert({
|
||||
type: 'danger',
|
||||
message: err.message
|
||||
})))
|
||||
if (!web.LoggedIn()) {
|
||||
let url = `${window.location.origin}/minio/download/${currentBucket}/${encPrefix}?token=''`
|
||||
window.location = url
|
||||
} else {
|
||||
// Download the selected file.
|
||||
web.CreateURLToken()
|
||||
.then(res => {
|
||||
let url = `${window.location.origin}${minioBrowserPrefix}/download/${currentBucket}/${encPrefix}?token=${res.token}`
|
||||
window.location = url
|
||||
})
|
||||
.catch(err => dispatch(actions.showAlert({
|
||||
type: 'danger',
|
||||
message: err.message
|
||||
})))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -421,18 +426,22 @@ export default class Browse extends React.Component {
|
||||
objects: this.props.checkedObjects,
|
||||
prefix: this.props.currentPath
|
||||
}
|
||||
|
||||
web.CreateURLToken()
|
||||
.then(res => {
|
||||
let requestUrl = location.origin + "/minio/zip?token=" + res.token
|
||||
|
||||
this.xhr = new XMLHttpRequest()
|
||||
dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
|
||||
})
|
||||
.catch(err => dispatch(actions.showAlert({
|
||||
type: 'danger',
|
||||
message: err.message
|
||||
})))
|
||||
if (!web.LoggedIn()) {
|
||||
let requestUrl = location.origin + "/minio/zip?token=''"
|
||||
this.xhr = new XMLHttpRequest()
|
||||
dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
|
||||
} else {
|
||||
web.CreateURLToken()
|
||||
.then(res => {
|
||||
let requestUrl = location.origin + minioBrowserPrefix + "/zip?token=" + res.token
|
||||
this.xhr = new XMLHttpRequest()
|
||||
dispatch(actions.downloadSelected(requestUrl, req, this.xhr))
|
||||
})
|
||||
.catch(err => dispatch(actions.showAlert({
|
||||
type: 'danger',
|
||||
message: err.message
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
clearSelected() {
|
||||
@@ -493,7 +502,7 @@ export default class Browse extends React.Component {
|
||||
settingsFunc={ this.showSettings.bind(this) }
|
||||
logoutFunc={ this.logout.bind(this) } />
|
||||
} else {
|
||||
loginButton = <a className='btn btn-danger' href='/minio/login'>Login</a>
|
||||
loginButton = <a className='btn btn-danger' href={minioBrowserPrefix+'/login'}>Login</a>
|
||||
}
|
||||
|
||||
if (web.LoggedIn()) {
|
||||
|
||||
@@ -17,7 +17,9 @@
|
||||
// File for all the browser constants.
|
||||
|
||||
// minioBrowserPrefix absolute path.
|
||||
export const minioBrowserPrefix = '/minio'
|
||||
var p = window.location.pathname
|
||||
export const minioBrowserPrefix = p.slice(0, p.indexOf("/", 1))
|
||||
|
||||
export const READ_ONLY = 'readonly'
|
||||
export const WRITE_ONLY = 'writeonly'
|
||||
export const READ_WRITE = 'readwrite'
|
||||
|
||||
+13
-15
@@ -35,8 +35,6 @@
|
||||
"esformatter": "^0.10.0",
|
||||
"esformatter-jsx": "^7.4.1",
|
||||
"esformatter-jsx-ignore": "^1.0.6",
|
||||
"expect": "^1.20.2",
|
||||
"history": "^1.17.0",
|
||||
"html-webpack-plugin": "^2.22.0",
|
||||
"json-loader": "^0.5.4",
|
||||
"karma": "^0.13.22",
|
||||
@@ -48,38 +46,38 @@
|
||||
"less": "^2.7.1",
|
||||
"less-loader": "^2.2.3",
|
||||
"mocha": "^2.5.3",
|
||||
"moment": "^2.15.1",
|
||||
"purifycss-webpack-plugin": "^2.0.3",
|
||||
"react": "^0.14.8",
|
||||
"react-addons-test-utils": "^0.14.8",
|
||||
"react-bootstrap": "^0.28.5",
|
||||
"react-custom-scrollbars": "^2.3.0",
|
||||
"react-redux": "^4.4.5",
|
||||
"react-router": "^2.8.1",
|
||||
"redux": "^3.6.0",
|
||||
"redux-thunk": "^1.0.3",
|
||||
"style-loader": "^0.13.1",
|
||||
"superagent": "^1.8.4",
|
||||
"superagent-es6-promise": "^1.0.0",
|
||||
"url-loader": "^0.5.7",
|
||||
"webpack": "^1.12.11",
|
||||
"webpack-dev-server": "^1.14.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"bootstrap": "^3.3.6",
|
||||
"classnames": "^2.2.3",
|
||||
"expect": "^1.20.2",
|
||||
"font-awesome": "^4.7.0",
|
||||
"history": "^1.17.0",
|
||||
"humanize": "0.0.9",
|
||||
"json-loader": "^0.5.4",
|
||||
"local-storage-fallback": "^1.3.0",
|
||||
"mime-db": "^1.25.0",
|
||||
"mime-types": "^2.1.13",
|
||||
"moment": "^2.15.1",
|
||||
"react": "^0.14.8",
|
||||
"react-addons-test-utils": "^0.14.8",
|
||||
"react-bootstrap": "^0.28.5",
|
||||
"react-copy-to-clipboard": "^4.2.3",
|
||||
"react-custom-scrollbars": "^2.2.2",
|
||||
"react-dom": "^0.14.6",
|
||||
"react-dropzone": "^3.5.3",
|
||||
"react-infinite-scroller": "^1.0.6",
|
||||
"react-onclickout": "2.0.4"
|
||||
"react-onclickout": "2.0.4",
|
||||
"react-redux": "^4.4.5",
|
||||
"react-router": "^2.8.1",
|
||||
"redux": "^3.6.0",
|
||||
"redux-thunk": "^1.0.3",
|
||||
"superagent": "^1.8.4",
|
||||
"superagent-es6-promise": "^1.0.0",
|
||||
"webpack": "^1.12.11"
|
||||
}
|
||||
}
|
||||
|
||||
+64
-67
File diff suppressed because one or more lines are too long
@@ -1,110 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
_init() {
|
||||
# Save release LDFLAGS
|
||||
LDFLAGS=$(go run buildscripts/gen-ldflags.go)
|
||||
|
||||
# Extract release tag
|
||||
release_tag=$(echo $LDFLAGS | awk {'print $6'} | cut -f2 -d=)
|
||||
|
||||
# Verify release tag.
|
||||
if [ -z "$release_tag" ]; then
|
||||
echo "Release tag cannot be empty. Please check return value of \`go run buildscripts/gen-ldflags.go\`"
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
# Extract release string.
|
||||
release_str=$(echo $MINIO_RELEASE | tr '[:upper:]' '[:lower:]')
|
||||
|
||||
# Verify release string.
|
||||
if [ -z "$release_str" ]; then
|
||||
echo "Release string cannot be empty. Please set \`MINIO_RELEASE\` env variable."
|
||||
exit 1;
|
||||
fi
|
||||
|
||||
# List of supported architectures
|
||||
SUPPORTED_OSARCH='linux/amd64 windows/amd64 darwin/amd64'
|
||||
|
||||
## System binaries
|
||||
CP=`which cp`
|
||||
SHASUM=`which shasum`
|
||||
SHA256SUM="${SHASUM} -a 256"
|
||||
SED=`which sed`
|
||||
}
|
||||
|
||||
go_build() {
|
||||
local osarch=$1
|
||||
os=$(echo $osarch | cut -f1 -d'/')
|
||||
arch=$(echo $osarch | cut -f2 -d'/')
|
||||
package=$(go list -f '{{.ImportPath}}')
|
||||
echo -n "-->"
|
||||
printf "%15s:%s\n" "${osarch}" "${package}"
|
||||
|
||||
# Release binary name
|
||||
release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
|
||||
# Release binary downloadable name
|
||||
release_real_bin="$release_str/$os-$arch/$(basename $package)"
|
||||
|
||||
# Release sha1sum name
|
||||
release_shasum="$release_str/$os-$arch/$(basename $package).${release_tag}.shasum"
|
||||
# Release sha1sum default
|
||||
release_shasum_default="$release_str/$os-$arch/$(basename $package).shasum"
|
||||
|
||||
# Release sha256sum name
|
||||
release_sha256sum="$release_str/$os-$arch/$(basename $package).${release_tag}.sha256sum"
|
||||
# Release sha256sum default
|
||||
release_sha256sum_default="$release_str/$os-$arch/$(basename $package).sha256sum"
|
||||
|
||||
# Go build to build the binary.
|
||||
CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
|
||||
|
||||
# Create copy
|
||||
if [ $os == "windows" ]; then
|
||||
$CP -p $release_bin ${release_real_bin}.exe
|
||||
else
|
||||
$CP -p $release_bin $release_real_bin
|
||||
fi
|
||||
|
||||
# Calculate sha1sum
|
||||
shasum_str=$(${SHASUM} ${release_bin})
|
||||
echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
|
||||
$CP -p $release_shasum $release_shasum_default
|
||||
|
||||
# Calculate sha256sum
|
||||
sha256sum_str=$(${SHA256SUM} ${release_bin})
|
||||
echo ${sha256sum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_sha256sum
|
||||
$CP -p $release_sha256sum $release_sha256sum_default
|
||||
}
|
||||
|
||||
main() {
|
||||
# Build releases.
|
||||
echo "Executing $release_str builds for OS: ${SUPPORTED_OSARCH}"
|
||||
echo "Choose an OS Arch from the below"
|
||||
for osarch in ${SUPPORTED_OSARCH}; do
|
||||
echo ${osarch}
|
||||
done
|
||||
|
||||
read -p "If you want to build for all, Just press Enter: " chosen_osarch
|
||||
if [ "$chosen_osarch" = "" ] || [ "$chosen_osarch" = "all" ]; then
|
||||
for each_osarch in ${SUPPORTED_OSARCH}; do
|
||||
go_build ${each_osarch}
|
||||
done
|
||||
else
|
||||
local found=0
|
||||
for each_osarch in ${SUPPORTED_OSARCH}; do
|
||||
if [ "$chosen_osarch" = "$each_osarch" ]; then
|
||||
found=1
|
||||
fi
|
||||
done
|
||||
if [ ${found} -eq 1 ]; then
|
||||
go_build ${chosen_osarch}
|
||||
else
|
||||
echo "Unknown architecture \"${chosen_osarch}\""
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
# Run main.
|
||||
_init && main
|
||||
@@ -21,7 +21,7 @@ _init() {
|
||||
|
||||
## Minimum required versions for build dependencies
|
||||
GIT_VERSION="1.0"
|
||||
GO_VERSION="1.8.3"
|
||||
GO_VERSION="1.9.1"
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
|
||||
@@ -75,6 +75,16 @@ function start_minio_dist()
|
||||
echo "${minio_pids[@]}"
|
||||
}
|
||||
|
||||
function start_minio_gateway_s3()
|
||||
{
|
||||
MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG \
|
||||
"${MINIO[@]}" gateway s3 https://play.minio.io:9000 >"$WORK_DIR/minio-gateway-s3.log" 2>&1 &
|
||||
minio_pid=$!
|
||||
sleep 3
|
||||
|
||||
echo "$minio_pid"
|
||||
}
|
||||
|
||||
function run_test_fs()
|
||||
{
|
||||
minio_pid="$(start_minio_fs)"
|
||||
@@ -86,9 +96,9 @@ function run_test_fs()
|
||||
sleep 3
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
cat fs-minio.log
|
||||
cat "$WORK_DIR/fs-minio.log"
|
||||
fi
|
||||
rm -f fs-minio.log
|
||||
rm -f "$WORK_DIR/fs-minio.log"
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
@@ -104,9 +114,9 @@ function run_test_xl()
|
||||
sleep 3
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
cat xl-minio.log
|
||||
cat "$WORK_DIR/xl-minio.log"
|
||||
fi
|
||||
rm -f xl-minio.log
|
||||
rm -f "$WORK_DIR/xl-minio.log"
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
@@ -125,20 +135,40 @@ function run_test_dist()
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
echo "server1 log:"
|
||||
cat dist-minio-9000.log
|
||||
cat "$WORK_DIR/dist-minio-9000.log"
|
||||
echo "server2 log:"
|
||||
cat dist-minio-9001.log
|
||||
cat "$WORK_DIR/dist-minio-9001.log"
|
||||
echo "server3 log:"
|
||||
cat dist-minio-9002.log
|
||||
cat "$WORK_DIR/dist-minio-9002.log"
|
||||
echo "server4 log:"
|
||||
cat dist-minio-9003.log
|
||||
cat "$WORK_DIR/dist-minio-9003.log"
|
||||
fi
|
||||
|
||||
rm -f dist-minio-900{0..3}.log
|
||||
rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function run_test_gateway_s3()
|
||||
{
|
||||
minio_pid="$(start_minio_gateway_s3)"
|
||||
|
||||
export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||
rv=$?
|
||||
|
||||
kill "$minio_pid"
|
||||
sleep 3
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
cat "$WORK_DIR/minio-gateway-s3.log"
|
||||
fi
|
||||
rm -f "$WORK_DIR/minio-gateway-s3.log"
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function __init__()
|
||||
{
|
||||
echo "Initializing environment"
|
||||
@@ -146,15 +176,16 @@ function __init__()
|
||||
mkdir -p "$MINIO_CONFIG_DIR"
|
||||
mkdir -p "$MINT_DATA_DIR"
|
||||
|
||||
if ! wget -q -O "$WORK_DIR/mc" https://dl.minio.io/client/mc/release/linux-amd64/mc; then
|
||||
echo "failed to download https://dl.minio.io/client/mc/release/linux-amd64/mc"
|
||||
if ! go get -u github.com/minio/mc; then
|
||||
echo "failed to download https://github.com/minio/mc"
|
||||
exit 1
|
||||
fi
|
||||
/bin/cp -a "$(go env GOPATH)"/bin/mc "$WORK_DIR/mc"
|
||||
|
||||
chmod a+x "$WORK_DIR/mc"
|
||||
|
||||
base64 /dev/urandom | head -c 1048576 >"$FILE_1_MB"
|
||||
base64 /dev/urandom | head -c 68157440 >"$FILE_65_MB"
|
||||
shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
|
||||
shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
|
||||
|
||||
## version is purposefully set to '3' for minio to migrate configuration file
|
||||
echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
|
||||
@@ -171,21 +202,28 @@ function main()
|
||||
{
|
||||
echo "Testing in FS setup"
|
||||
if ! run_test_fs; then
|
||||
echo "running test for FS setup failed"
|
||||
echo "FAILED"
|
||||
rm -fr "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in XL setup"
|
||||
if ! run_test_xl; then
|
||||
echo "running test for XL setup"
|
||||
echo "FAILED"
|
||||
rm -fr "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Distribute XL setup"
|
||||
if ! run_test_dist; then
|
||||
echo "running test for Distribute setup"
|
||||
echo "FAILED"
|
||||
rm -fr "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Gateway S3 setup"
|
||||
if ! run_test_gateway_s3; then
|
||||
echo "FAILED"
|
||||
rm -fr "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
+10
-4
@@ -28,6 +28,8 @@ import (
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -168,7 +170,7 @@ func (adminAPI adminAPIHandlers) ServiceCredentialsHandler(w http.ResponseWriter
|
||||
return
|
||||
}
|
||||
|
||||
creds, err := createCredential(req.Username, req.Password)
|
||||
creds, err := auth.CreateCredentials(req.Username, req.Password)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
@@ -838,15 +840,19 @@ func (adminAPI adminAPIHandlers) HealFormatHandler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
|
||||
// Wrap into retrying disks
|
||||
retryingDisks := initRetryableStorageDisks(bootstrapDisks,
|
||||
time.Millisecond, time.Millisecond*5)
|
||||
|
||||
// Heal format.json on available storage.
|
||||
err = healFormatXL(bootstrapDisks)
|
||||
err = healFormatXL(retryingDisks)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Instantiate new object layer with newly formatted storage.
|
||||
newObjectAPI, err := newXLObjects(bootstrapDisks)
|
||||
newObjectAPI, err := newXLObjects(retryingDisks)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
@@ -979,7 +985,7 @@ func (adminAPI adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
var config serverConfigV19
|
||||
var config serverConfigV20
|
||||
err = json.Unmarshal(configBytes, &config)
|
||||
|
||||
if err != nil {
|
||||
|
||||
@@ -32,6 +32,7 @@ import (
|
||||
"time"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
var configJSON = []byte(`{
|
||||
@@ -263,7 +264,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
|
||||
|
||||
// getServiceCmdRequest - Constructs a management REST API request for service
|
||||
// subcommands for a given cmdType value.
|
||||
func getServiceCmdRequest(cmd cmdType, cred credential, body []byte) (*http.Request, error) {
|
||||
func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*http.Request, error) {
|
||||
req, err := newTestRequest(cmd.apiMethod(), "/?service", 0, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -943,7 +944,7 @@ func TestHealObjectHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
_, err = adminTestBed.objLayer.PutObject(bucketName, objName,
|
||||
NewHashReader(bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""), nil)
|
||||
mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""), nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create %s - %v", objName, err)
|
||||
}
|
||||
@@ -1083,7 +1084,7 @@ func TestHealUploadHandler(t *testing.T) {
|
||||
// Upload a part.
|
||||
partID := 1
|
||||
_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
|
||||
partID, NewHashReader(bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""))
|
||||
partID, mustGetHashReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "", ""))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
|
||||
bucketName, objName, err)
|
||||
|
||||
+116
-14
@@ -19,6 +19,9 @@ package cmd
|
||||
import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// APIError structure
|
||||
@@ -114,12 +117,23 @@ const (
|
||||
ErrInvalidQueryParams
|
||||
ErrBucketAlreadyOwnedByYou
|
||||
ErrInvalidDuration
|
||||
ErrNotSupported
|
||||
ErrBucketAlreadyExists
|
||||
ErrMetadataTooLarge
|
||||
ErrUnsupportedMetadata
|
||||
ErrMaximumExpires
|
||||
// Add new error codes here.
|
||||
|
||||
// Server-Side-Encryption (with Customer provided key) related API errors.
|
||||
|
||||
ErrInsecureSSECustomerRequest
|
||||
ErrSSEEncryptedObject
|
||||
ErrInvalidEncryptionParameters
|
||||
ErrInvalidSSECustomerAlgorithm
|
||||
ErrInvalidSSECustomerKey
|
||||
ErrMissingSSECustomerKey
|
||||
ErrMissingSSECustomerKeyMD5
|
||||
ErrSSECustomerKeyMD5Mismatch
|
||||
|
||||
// Bucket notification related errors.
|
||||
ErrEventNotification
|
||||
ErrARNNotification
|
||||
@@ -147,6 +161,8 @@ const (
|
||||
ErrInvalidResourceName
|
||||
ErrServerNotInitialized
|
||||
ErrOperationTimedOut
|
||||
ErrPartsSizeUnequal
|
||||
ErrInvalidRequest
|
||||
// Add new extended error codes here.
|
||||
// Please open a https://github.com/minio/minio/issues before adding
|
||||
// new error codes here.
|
||||
@@ -156,6 +172,7 @@ const (
|
||||
ErrAdminConfigNoQuorum
|
||||
ErrAdminCredentialsMismatch
|
||||
ErrInsecureClientRequest
|
||||
ErrObjectTampered
|
||||
)
|
||||
|
||||
// error code to APIError structure, these fields carry respective
|
||||
@@ -571,6 +588,51 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Range specified is not valid for source object",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMetadataTooLarge: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Your metadata headers exceed the maximum allowed metadata size.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInsecureSSECustomerRequest: {
|
||||
Code: "InvalidRequest",
|
||||
Description: errInsecureSSERequest.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrSSEEncryptedObject: {
|
||||
Code: "InvalidRequest",
|
||||
Description: errEncryptedObject.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidEncryptionParameters: {
|
||||
Code: "InvalidRequest",
|
||||
Description: "The encryption parameters are not applicable to this object.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidSSECustomerAlgorithm: {
|
||||
Code: "InvalidArgument",
|
||||
Description: errInvalidSSEAlgorithm.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidSSECustomerKey: {
|
||||
Code: "InvalidArgument",
|
||||
Description: errInvalidSSEKey.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingSSECustomerKey: {
|
||||
Code: "InvalidArgument",
|
||||
Description: errMissingSSEKey.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrMissingSSECustomerKeyMD5: {
|
||||
Code: "InvalidArgument",
|
||||
Description: errMissingSSEKeyMD5.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrSSECustomerKeyMD5Mismatch: {
|
||||
Code: "InvalidArgument",
|
||||
Description: errSSEKeyMD5Mismatch.Error(),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
|
||||
/// S3 extensions.
|
||||
ErrContentSHA256Mismatch: {
|
||||
@@ -650,16 +712,35 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "A timeout occurred while trying to lock a resource",
|
||||
HTTPStatusCode: http.StatusRequestTimeout,
|
||||
},
|
||||
ErrMetadataTooLarge: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Your metadata headers exceed the maximum allowed metadata size.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrUnsupportedMetadata: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Your metadata headers are not supported.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrPartsSizeUnequal: {
|
||||
Code: "XMinioPartsSizeUnequal",
|
||||
Description: "All parts except the last part should be of the same size.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrObjectTampered: {
|
||||
Code: "XMinioObjectTampered",
|
||||
Description: errObjectTampered.Error(),
|
||||
HTTPStatusCode: http.StatusPartialContent,
|
||||
},
|
||||
ErrMaximumExpires: {
|
||||
Code: "AuthorizationQueryParametersError",
|
||||
Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// Generic Invalid-Request error. Should be used for response errors only for unlikely
|
||||
// corner case errors for which introducing new APIErrorCode is not worth it. errorIf()
|
||||
// should be used to log the error at the source of the error for debugging purposes.
|
||||
ErrInvalidRequest: {
|
||||
Code: "InvalidRequest",
|
||||
Description: "Invalid Request",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
|
||||
// Add your error structure here.
|
||||
}
|
||||
|
||||
@@ -676,15 +757,13 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
switch err {
|
||||
case errSignatureMismatch:
|
||||
apiErr = ErrSignatureDoesNotMatch
|
||||
case errContentSHA256Mismatch:
|
||||
apiErr = ErrContentSHA256Mismatch
|
||||
case errDataTooLarge:
|
||||
apiErr = ErrEntityTooLarge
|
||||
case errDataTooSmall:
|
||||
apiErr = ErrEntityTooSmall
|
||||
case errInvalidAccessKeyLength:
|
||||
case auth.ErrInvalidAccessKeyLength:
|
||||
apiErr = ErrAdminInvalidAccessKey
|
||||
case errInvalidSecretKeyLength:
|
||||
case auth.ErrInvalidSecretKeyLength:
|
||||
apiErr = ErrAdminInvalidSecretKey
|
||||
}
|
||||
|
||||
@@ -693,10 +772,31 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
return apiErr
|
||||
}
|
||||
|
||||
switch err { // SSE errors
|
||||
case errInsecureSSERequest:
|
||||
return ErrInsecureSSECustomerRequest
|
||||
case errInvalidSSEAlgorithm:
|
||||
return ErrInvalidSSECustomerAlgorithm
|
||||
case errInvalidSSEKey:
|
||||
return ErrInvalidSSECustomerKey
|
||||
case errMissingSSEKey:
|
||||
return ErrMissingSSECustomerKey
|
||||
case errMissingSSEKeyMD5:
|
||||
return ErrMissingSSECustomerKeyMD5
|
||||
case errSSEKeyMD5Mismatch:
|
||||
return ErrSSECustomerKeyMD5Mismatch
|
||||
case errObjectTampered:
|
||||
return ErrObjectTampered
|
||||
case errEncryptedObject:
|
||||
return ErrSSEEncryptedObject
|
||||
case errSSEKeyMismatch:
|
||||
return ErrAccessDenied // no access without correct key
|
||||
}
|
||||
|
||||
switch err.(type) {
|
||||
case StorageFull:
|
||||
apiErr = ErrStorageFull
|
||||
case BadDigest:
|
||||
case hash.BadDigest:
|
||||
apiErr = ErrBadDigest
|
||||
case AllAccessDisabled:
|
||||
apiErr = ErrAllAccessDisabled
|
||||
@@ -742,14 +842,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrEntityTooSmall
|
||||
case SignatureDoesNotMatch:
|
||||
apiErr = ErrSignatureDoesNotMatch
|
||||
case SHA256Mismatch:
|
||||
case hash.SHA256Mismatch:
|
||||
apiErr = ErrContentSHA256Mismatch
|
||||
case ObjectTooLarge:
|
||||
apiErr = ErrEntityTooLarge
|
||||
case ObjectTooSmall:
|
||||
apiErr = ErrEntityTooSmall
|
||||
case NotSupported:
|
||||
apiErr = ErrNotSupported
|
||||
case NotImplemented:
|
||||
apiErr = ErrNotImplemented
|
||||
case PolicyNotFound:
|
||||
@@ -758,6 +856,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrEntityTooLarge
|
||||
case UnsupportedMetadata:
|
||||
apiErr = ErrUnsupportedMetadata
|
||||
case PartsSizeUnequal:
|
||||
apiErr = ErrPartsSizeUnequal
|
||||
case BucketPolicyNotFound:
|
||||
apiErr = ErrNoSuchBucketPolicy
|
||||
default:
|
||||
apiErr = ErrInternalError
|
||||
}
|
||||
|
||||
+43
-113
@@ -19,122 +19,52 @@ package cmd
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
var toAPIErrorCodeTests = []struct {
|
||||
err error
|
||||
errCode APIErrorCode
|
||||
}{
|
||||
{err: hash.BadDigest{}, errCode: ErrBadDigest},
|
||||
{err: hash.SHA256Mismatch{}, errCode: ErrContentSHA256Mismatch},
|
||||
{err: IncompleteBody{}, errCode: ErrIncompleteBody},
|
||||
{err: ObjectExistsAsDirectory{}, errCode: ErrObjectExistsAsDirectory},
|
||||
{err: BucketNameInvalid{}, errCode: ErrInvalidBucketName},
|
||||
{err: BucketExists{}, errCode: ErrBucketAlreadyOwnedByYou},
|
||||
{err: ObjectNotFound{}, errCode: ErrNoSuchKey},
|
||||
{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
|
||||
{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrReadQuorum},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrWriteQuorum},
|
||||
{err: UnsupportedDelimiter{}, errCode: ErrNotImplemented},
|
||||
{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
|
||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||
{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
|
||||
{err: BucketNotEmpty{}, errCode: ErrBucketNotEmpty},
|
||||
{err: BucketNotFound{}, errCode: ErrNoSuchBucket},
|
||||
{err: StorageFull{}, errCode: ErrStorageFull},
|
||||
{err: NotImplemented{}, errCode: ErrNotImplemented},
|
||||
{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
|
||||
|
||||
// SSE-C errors
|
||||
{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
|
||||
{err: errInvalidSSEAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
|
||||
{err: errMissingSSEKey, errCode: ErrMissingSSECustomerKey},
|
||||
{err: errInvalidSSEKey, errCode: ErrInvalidSSECustomerKey},
|
||||
{err: errMissingSSEKeyMD5, errCode: ErrMissingSSECustomerKeyMD5},
|
||||
{err: errSSEKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch},
|
||||
{err: errObjectTampered, errCode: ErrObjectTampered},
|
||||
|
||||
{err: nil, errCode: ErrNone},
|
||||
{err: errors.New("Custom error"), errCode: ErrInternalError}, // Case where err type is unknown.
|
||||
}
|
||||
|
||||
func TestAPIErrCode(t *testing.T) {
|
||||
testCases := []struct {
|
||||
err error
|
||||
errCode APIErrorCode
|
||||
}{
|
||||
// Valid cases.
|
||||
{
|
||||
BadDigest{},
|
||||
ErrBadDigest,
|
||||
},
|
||||
{
|
||||
IncompleteBody{},
|
||||
ErrIncompleteBody,
|
||||
},
|
||||
{
|
||||
ObjectExistsAsDirectory{},
|
||||
ErrObjectExistsAsDirectory,
|
||||
},
|
||||
{
|
||||
BucketNameInvalid{},
|
||||
ErrInvalidBucketName,
|
||||
},
|
||||
{
|
||||
BucketExists{},
|
||||
ErrBucketAlreadyOwnedByYou,
|
||||
},
|
||||
{
|
||||
ObjectNotFound{},
|
||||
ErrNoSuchKey,
|
||||
},
|
||||
{
|
||||
ObjectNameInvalid{},
|
||||
ErrInvalidObjectName,
|
||||
},
|
||||
{
|
||||
InvalidUploadID{},
|
||||
ErrNoSuchUpload,
|
||||
},
|
||||
{
|
||||
InvalidPart{},
|
||||
ErrInvalidPart,
|
||||
},
|
||||
{
|
||||
InsufficientReadQuorum{},
|
||||
ErrReadQuorum,
|
||||
},
|
||||
{
|
||||
InsufficientWriteQuorum{},
|
||||
ErrWriteQuorum,
|
||||
},
|
||||
{
|
||||
UnsupportedDelimiter{},
|
||||
ErrNotImplemented,
|
||||
},
|
||||
{
|
||||
InvalidMarkerPrefixCombination{},
|
||||
ErrNotImplemented,
|
||||
},
|
||||
{
|
||||
InvalidUploadIDKeyCombination{},
|
||||
ErrNotImplemented,
|
||||
},
|
||||
{
|
||||
MalformedUploadID{},
|
||||
ErrNoSuchUpload,
|
||||
},
|
||||
{
|
||||
PartTooSmall{},
|
||||
ErrEntityTooSmall,
|
||||
},
|
||||
{
|
||||
BucketNotEmpty{},
|
||||
ErrBucketNotEmpty,
|
||||
},
|
||||
{
|
||||
BucketNotFound{},
|
||||
ErrNoSuchBucket,
|
||||
},
|
||||
{
|
||||
StorageFull{},
|
||||
ErrStorageFull,
|
||||
},
|
||||
{
|
||||
NotSupported{},
|
||||
ErrNotSupported,
|
||||
},
|
||||
{
|
||||
NotImplemented{},
|
||||
ErrNotImplemented,
|
||||
},
|
||||
{
|
||||
errSignatureMismatch,
|
||||
ErrSignatureDoesNotMatch,
|
||||
},
|
||||
{
|
||||
errContentSHA256Mismatch,
|
||||
ErrContentSHA256Mismatch,
|
||||
}, // End of all valid cases.
|
||||
|
||||
// Case where err is nil.
|
||||
{
|
||||
nil,
|
||||
ErrNone,
|
||||
},
|
||||
|
||||
// Case where err type is unknown.
|
||||
{
|
||||
errors.New("Custom error"),
|
||||
ErrInternalError,
|
||||
},
|
||||
}
|
||||
|
||||
// Validate all the errors with their API error codes.
|
||||
for i, testCase := range testCases {
|
||||
for i, testCase := range toAPIErrorCodeTests {
|
||||
errCode := toAPIErrorCode(testCase.err)
|
||||
if errCode != testCase.errCode {
|
||||
t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)
|
||||
|
||||
@@ -70,6 +70,14 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
|
||||
w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
|
||||
}
|
||||
|
||||
if objInfo.ContentType != "" {
|
||||
w.Header().Set("Content-Type", objInfo.ContentType)
|
||||
}
|
||||
|
||||
if objInfo.ContentEncoding != "" {
|
||||
w.Header().Set("Content-Encoding", objInfo.ContentEncoding)
|
||||
}
|
||||
|
||||
// Set all other user defined metadata.
|
||||
for k, v := range objInfo.UserDefined {
|
||||
w.Header().Set(k, v)
|
||||
|
||||
+1
-1
@@ -307,7 +307,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
||||
for _, bucket := range buckets {
|
||||
var listbucket = Bucket{}
|
||||
listbucket.Name = bucket.Name
|
||||
listbucket.CreationDate = bucket.Created.Format(timeFormatAMZLong)
|
||||
listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong)
|
||||
listbucket.HealBucketInfo = bucket.HealBucketInfo
|
||||
listbuckets = append(listbuckets, listbucket)
|
||||
}
|
||||
|
||||
+66
-60
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import router "github.com/gorilla/mux"
|
||||
import "net/http"
|
||||
|
||||
// objectAPIHandler implements and provides http handlers for S3 API.
|
||||
type objectAPIHandlers struct {
|
||||
@@ -32,70 +33,75 @@ func registerAPIRouter(mux *router.Router) {
|
||||
|
||||
// API Router
|
||||
apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
|
||||
var routers []*router.Router
|
||||
if globalDomainName != "" {
|
||||
routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
|
||||
}
|
||||
routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
|
||||
|
||||
// Bucket router
|
||||
bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
|
||||
for _, bucket := range routers {
|
||||
// Object operations
|
||||
// HeadObject
|
||||
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
|
||||
// CopyObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// PutObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// ListObjectPxarts
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// NewMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// GetObject
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
|
||||
// CopyObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
|
||||
// PutObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
|
||||
// DeleteObject
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
|
||||
|
||||
/// Object operations
|
||||
|
||||
// HeadObject
|
||||
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
|
||||
// CopyObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// PutObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// ListObjectPxarts
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// NewMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// GetObject
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
|
||||
// CopyObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
|
||||
// PutObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
|
||||
// DeleteObject
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
|
||||
|
||||
/// Bucket operations
|
||||
|
||||
// GetBucketLocation
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
|
||||
// GetBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
|
||||
// ListenBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("events", "{events:.*}")
|
||||
// ListMultipartUploads
|
||||
bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
|
||||
// ListObjectsV2
|
||||
bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
|
||||
// PutBucketPolicy
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
|
||||
// PutBucketNotification
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
|
||||
// PutBucket
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
|
||||
// HeadBucket
|
||||
bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
|
||||
// PostPolicy
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
|
||||
/// Bucket operations
|
||||
// GetBucketLocation
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
|
||||
// GetBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
|
||||
// ListenBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
|
||||
// ListMultipartUploads
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
|
||||
// ListObjectsV2
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
|
||||
// PutBucketPolicy
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
|
||||
// PutBucketNotification
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
|
||||
// PutBucket
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
|
||||
// HeadBucket
|
||||
bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
|
||||
// PostPolicy
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
|
||||
}
|
||||
|
||||
/// Root operation
|
||||
|
||||
// ListBuckets
|
||||
apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
|
||||
apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
|
||||
|
||||
// If none of the routes match.
|
||||
apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
}
|
||||
|
||||
+5
-1
@@ -127,7 +127,11 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
|
||||
if reqAuthType == authTypeAnonymous && policyAction != "" {
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
|
||||
sourceIP := getSourceIPAddress(r)
|
||||
return enforceBucketPolicy(bucket, policyAction, r.URL.Path,
|
||||
resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
|
||||
if err != nil {
|
||||
return ErrInternalError
|
||||
}
|
||||
return enforceBucketPolicy(bucket, policyAction, resource,
|
||||
r.Referer(), sourceIP, r.URL.Query())
|
||||
}
|
||||
|
||||
|
||||
@@ -23,6 +23,8 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// Test get request auth type.
|
||||
@@ -327,7 +329,7 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
}
|
||||
defer os.RemoveAll(path)
|
||||
|
||||
creds, err := createCredential("myuser", "mypassword")
|
||||
creds, err := auth.CreateCredentials("myuser", "mypassword")
|
||||
if err != nil {
|
||||
t.Fatalf("unable create credential, %s", err)
|
||||
}
|
||||
|
||||
+53
-39
@@ -50,20 +50,23 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
metadata := make(map[string]string)
|
||||
metadata["etag"] = getMD5Hash(textData)
|
||||
sha256sum := ""
|
||||
|
||||
md5hex := getMD5Hash(textData)
|
||||
sha256hex := ""
|
||||
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
// the actual benchmark for PutObject starts here. Reset the benchmark timer.
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), NewHashReader(bytes.NewBuffer(textData), int64(len(textData)), metadata["etag"], sha256sum), metadata)
|
||||
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
|
||||
if objInfo.ETag != md5hex {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
|
||||
}
|
||||
}
|
||||
// Benchmark ends here. Stop timer.
|
||||
@@ -94,13 +97,14 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for NewMultipartUpload.
|
||||
metadata := make(map[string]string)
|
||||
metadata["etag"] = getMD5Hash(textData)
|
||||
sha256sum := ""
|
||||
uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
md5hex := getMD5Hash(textData)
|
||||
sha256hex := ""
|
||||
|
||||
var textPartData []byte
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
@@ -115,15 +119,15 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
} else {
|
||||
textPartData = textData[j*partSize:]
|
||||
}
|
||||
metadata := make(map[string]string)
|
||||
metadata["etag"] = getMD5Hash([]byte(textPartData))
|
||||
md5hex = getMD5Hash([]byte(textPartData))
|
||||
var partInfo PartInfo
|
||||
partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, NewHashReader(bytes.NewBuffer(textPartData), int64(len(textPartData)), metadata["etag"], sha256sum))
|
||||
partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j,
|
||||
mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex))
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if partInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, metadata["etag"])
|
||||
if partInfo.ETag != md5hex {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, md5hex)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -205,23 +209,27 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
sha256sum := ""
|
||||
textData := generateBytesData(objSize)
|
||||
|
||||
// generate etag for the generated data.
|
||||
// etag of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
metadata := make(map[string]string)
|
||||
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
md5hex := getMD5Hash(textData)
|
||||
sha256hex := ""
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate etag for the generated data.
|
||||
// etag of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
metadata := make(map[string]string)
|
||||
metadata["etag"] = getMD5Hash(textData)
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), NewHashReader(bytes.NewBuffer(textData), int64(len(textData)), metadata["etag"], sha256sum), metadata)
|
||||
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
|
||||
if objInfo.ETag != md5hex {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,8 +326,10 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
metadata := make(map[string]string)
|
||||
metadata["etag"] = getMD5Hash([]byte(textData))
|
||||
sha256sum := ""
|
||||
|
||||
md5hex := getMD5Hash([]byte(textData))
|
||||
sha256hex := ""
|
||||
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
// the actual benchmark for PutObject starts here. Reset the benchmark timer.
|
||||
@@ -329,12 +339,13 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
i := 0
|
||||
for pb.Next() {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i), NewHashReader(bytes.NewBuffer(textData), int64(len(textData)), metadata["etag"], sha256sum), metadata)
|
||||
objInfo, err := obj.PutObject(bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, metadata["etag"])
|
||||
if objInfo.ETag != md5hex {
|
||||
b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, md5hex)
|
||||
}
|
||||
i++
|
||||
}
|
||||
@@ -361,23 +372,26 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
metadata := make(map[string]string)
|
||||
|
||||
md5hex := getMD5Hash([]byte(textData))
|
||||
sha256hex := ""
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
metadata := make(map[string]string)
|
||||
metadata["etag"] = getMD5Hash([]byte(textData))
|
||||
sha256sum := ""
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), NewHashReader(bytes.NewBuffer(textData), int64(len(textData)), metadata["etag"], sha256sum), metadata)
|
||||
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
|
||||
if objInfo.ETag != md5hex {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +21,8 @@ import (
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// SetAuthPeerArgs - Arguments collection for SetAuth RPC call
|
||||
@@ -29,7 +31,7 @@ type SetAuthPeerArgs struct {
|
||||
AuthRPCArgs
|
||||
|
||||
// New credentials that receiving peer should update to.
|
||||
Creds credential
|
||||
Creds auth.Credentials
|
||||
}
|
||||
|
||||
// SetAuthPeer - Update to new credentials sent from a peer Minio
|
||||
@@ -64,7 +66,7 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
|
||||
}
|
||||
|
||||
// Sends SetAuthPeer RPCs to all peers in the Minio cluster
|
||||
func updateCredsOnPeers(creds credential) map[string]error {
|
||||
func updateCredsOnPeers(creds auth.Credentials) map[string]error {
|
||||
// Get list of peer addresses (from globalS3Peers)
|
||||
peers := []string{}
|
||||
for _, p := range globalS3Peers {
|
||||
|
||||
@@ -19,6 +19,8 @@ package cmd
|
||||
import (
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// API suite container common to both FS and XL.
|
||||
@@ -29,8 +31,8 @@ type TestRPCBrowserPeerSuite struct {
|
||||
}
|
||||
|
||||
// Setting up the test suite and starting the Test server.
|
||||
func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
|
||||
s.testServer = StartTestBrowserPeerRPCServer(c, s.serverType)
|
||||
func (s *TestRPCBrowserPeerSuite) SetUpSuite(t *testing.T) {
|
||||
s.testServer = StartTestBrowserPeerRPCServer(t, s.serverType)
|
||||
s.testAuthConf = authConfig{
|
||||
serverAddr: s.testServer.Server.Listener.Addr().String(),
|
||||
accessKey: s.testServer.AccessKey,
|
||||
@@ -40,10 +42,9 @@ func (s *TestRPCBrowserPeerSuite) SetUpSuite(c *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// No longer used with gocheck, but used in explicit teardown code in
|
||||
// each test function. // Called implicitly by "gopkg.in/check.v1"
|
||||
// after all tests are run.
|
||||
func (s *TestRPCBrowserPeerSuite) TearDownSuite(c *testing.T) {
|
||||
// TeatDownSuite - called implicitly by after all tests are run in
|
||||
// browser peer rpc suite.
|
||||
func (s *TestRPCBrowserPeerSuite) TearDownSuite(t *testing.T) {
|
||||
s.testServer.Stop()
|
||||
}
|
||||
|
||||
@@ -62,7 +63,7 @@ func TestBrowserPeerRPC(t *testing.T) {
|
||||
// Tests for browser peer rpc.
|
||||
func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
|
||||
// Construct RPC call arguments.
|
||||
creds, err := createCredential("abcd1", "abcd1234")
|
||||
creds, err := auth.CreateCredentials("abcd1", "abcd1234")
|
||||
if err != nil {
|
||||
t.Fatalf("unable to create credential. %v", err)
|
||||
}
|
||||
|
||||
+32
-11
@@ -24,11 +24,15 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
mux "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/using-with-s3-actions.html
|
||||
@@ -55,8 +59,8 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
|
||||
}
|
||||
|
||||
// Fetch bucket policy, if policy is not set return access denied.
|
||||
policy := globalBucketPolicies.GetBucketPolicy(bucket)
|
||||
if policy == nil {
|
||||
p := globalBucketPolicies.GetBucketPolicy(bucket)
|
||||
if reflect.DeepEqual(p, emptyBucketPolicy) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
@@ -64,7 +68,7 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
|
||||
arn := bucketARNPrefix + strings.TrimSuffix(strings.TrimPrefix(resource, "/"), "/")
|
||||
|
||||
// Get conditions for policy verification.
|
||||
conditionKeyMap := make(map[string]set.StringSet)
|
||||
conditionKeyMap := make(policy.ConditionKeyMap)
|
||||
for queryParam := range queryParams {
|
||||
conditionKeyMap[queryParam] = set.CreateStringSet(queryParams.Get(queryParam))
|
||||
}
|
||||
@@ -77,7 +81,7 @@ func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, que
|
||||
conditionKeyMap["ip"] = set.CreateStringSet(sourceIP)
|
||||
|
||||
// Validate action, resource and conditions with current policy statements.
|
||||
if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, policy.Statements) {
|
||||
if !bucketPolicyEvalStatements(action, arn, conditionKeyMap, p.Statements) {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
return ErrNone
|
||||
@@ -89,14 +93,14 @@ func isBucketActionAllowed(action, bucket, prefix string) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
policy := globalBucketPolicies.GetBucketPolicy(bucket)
|
||||
if policy == nil {
|
||||
bp := globalBucketPolicies.GetBucketPolicy(bucket)
|
||||
if reflect.DeepEqual(bp, emptyBucketPolicy) {
|
||||
return false
|
||||
}
|
||||
resource := bucketARNPrefix + path.Join(bucket, prefix)
|
||||
var conditionKeyMap map[string]set.StringSet
|
||||
// Validate action, resource and conditions with current policy statements.
|
||||
return bucketPolicyEvalStatements(action, resource, conditionKeyMap, policy.Statements)
|
||||
return bucketPolicyEvalStatements(action, resource, conditionKeyMap, bp.Statements)
|
||||
}
|
||||
|
||||
// GetBucketLocationHandler - GET Bucket location.
|
||||
@@ -435,12 +439,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
bucket := mux.Vars(r)["bucket"]
|
||||
|
||||
// Require Content-Length to be set in the request
|
||||
size := r.ContentLength
|
||||
if size < 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
return
|
||||
}
|
||||
resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
// Make sure that the URL does not contain object name.
|
||||
if bucket != filepath.Clean(resource[1:]) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Here the parameter is the size of the form data that should
|
||||
// be loaded in memory, the remaining being put in temporary files.
|
||||
@@ -479,7 +495,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// Close multipart file
|
||||
defer fileBody.Close()
|
||||
|
||||
bucket := mux.Vars(r)["bucket"]
|
||||
formValues.Set("Bucket", bucket)
|
||||
|
||||
if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
|
||||
@@ -549,7 +564,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
return
|
||||
}
|
||||
sha256sum := ""
|
||||
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, object)
|
||||
if objectLock.GetLock(globalObjectTimeout) != nil {
|
||||
@@ -558,7 +572,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
defer objectLock.Unlock()
|
||||
|
||||
objInfo, err := objectAPI.PutObject(bucket, object, NewHashReader(fileBody, fileSize, metadata["etag"], sha256sum), metadata)
|
||||
hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to initialize hashReader.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
objInfo, err := objectAPI.PutObject(bucket, object, hashReader, metadata)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to create object.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
@@ -683,7 +704,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
_ = removeBucketPolicy(bucket, objectAPI)
|
||||
|
||||
// Notify all peers (including self) to update in-memory state
|
||||
S3PeersUpdateBucketPolicy(bucket, policyChange{true, nil})
|
||||
S3PeersUpdateBucketPolicy(bucket, policyChange{true, policy.BucketAccessPolicy{}})
|
||||
|
||||
// Delete notification config, if present - ignore any errors.
|
||||
_ = removeNotificationConfig(bucket, objectAPI)
|
||||
|
||||
@@ -24,6 +24,8 @@ import (
|
||||
"net/http/httptest"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// Wrapper for calling GetBucketPolicy HTTP handler tests for both XL multiple disks and single node setup.
|
||||
@@ -32,7 +34,7 @@ func TestGetBucketLocationHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
initBucketPolicies(obj)
|
||||
|
||||
// test cases with sample input and expected output.
|
||||
@@ -177,7 +179,7 @@ func TestHeadBucketHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
initBucketPolicies(obj)
|
||||
|
||||
// test cases with sample input and expected output.
|
||||
@@ -284,7 +286,7 @@ func TestListMultipartUploadsHandler(t *testing.T) {
|
||||
|
||||
// testListMultipartUploadsHandler - Tests validate listing of multipart uploads.
|
||||
func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
initBucketPolicies(obj)
|
||||
|
||||
// Collection of non-exhaustive ListMultipartUploads test cases, valid errors
|
||||
@@ -522,7 +524,7 @@ func TestListBucketsHandler(t *testing.T) {
|
||||
|
||||
// testListBucketsHandler - Tests validate listing of buckets.
|
||||
func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
bucketName string
|
||||
@@ -615,7 +617,7 @@ func TestAPIDeleteMultipleObjectsHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
initBucketPolicies(obj)
|
||||
|
||||
var err error
|
||||
@@ -631,7 +633,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
for i := 0; i < 10; i++ {
|
||||
objectName := "test-object-" + strconv.Itoa(i)
|
||||
// uploading the object.
|
||||
_, err = obj.PutObject(bucketName, objectName, NewHashReader(bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
|
||||
_, err = obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil)
|
||||
// if object upload fails stop the test.
|
||||
if err != nil {
|
||||
t.Fatalf("Put Object %d: Error uploading object: <ERROR> %v", i, err)
|
||||
@@ -805,7 +807,7 @@ func TestIsBucketActionAllowed(t *testing.T) {
|
||||
}
|
||||
|
||||
func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
// input.
|
||||
|
||||
@@ -137,13 +137,13 @@ type bucketMeta struct {
|
||||
|
||||
// Notification event object metadata.
|
||||
type objectMeta struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
ETag string `json:"eTag,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
UserDefined map[string]string `json:"userDefined,omitempty"`
|
||||
VersionID string `json:"versionId,omitempty"`
|
||||
Sequencer string `json:"sequencer"`
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
ETag string `json:"eTag,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
UserMetadata map[string]string `json:"userMetadata,omitempty"`
|
||||
VersionID string `json:"versionId,omitempty"`
|
||||
Sequencer string `json:"sequencer"`
|
||||
}
|
||||
|
||||
const (
|
||||
|
||||
@@ -22,7 +22,9 @@ import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
@@ -63,13 +65,13 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
|
||||
// Attempt to successfully load notification config.
|
||||
nConfig, err := loadNotificationConfig(bucket, objAPI)
|
||||
if err != nil && err != errNoSuchNotifications {
|
||||
if err != nil && errorCause(err) != errNoSuchNotifications {
|
||||
errorIf(err, "Unable to read notification configuration.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
// For no notifications we write a dummy XML.
|
||||
if err == errNoSuchNotifications {
|
||||
if errorCause(err) == errNoSuchNotifications {
|
||||
// Complies with the s3 behavior in this regard.
|
||||
nConfig = ¬ificationConfig{}
|
||||
}
|
||||
@@ -150,6 +152,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
// Convert the incoming ARNs properly to the GetRegion().
|
||||
for i, queueConfig := range notificationCfg.QueueConfigs {
|
||||
queueConfig.QueueARN = unmarshalSqsARN(queueConfig.QueueARN).String()
|
||||
notificationCfg.QueueConfigs[i] = queueConfig
|
||||
}
|
||||
|
||||
// Put bucket notification config.
|
||||
err = PutBucketNotificationConfig(bucket, ¬ificationCfg, objectAPI)
|
||||
if err != nil {
|
||||
@@ -207,14 +215,6 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
|
||||
return err
|
||||
}
|
||||
|
||||
// https://github.com/containous/traefik/issues/560
|
||||
// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
|
||||
//
|
||||
// Proxies might buffer the connection to avoid this we
|
||||
// need the proper MIME type before writing to client.
|
||||
// This MIME header tells the proxies to avoid buffering
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
|
||||
// Add additional CRLF characters for client to
|
||||
// differentiate the individual events properly.
|
||||
_, err = w.Write(append(notificationBytes, crlf...))
|
||||
@@ -226,25 +226,61 @@ func writeNotification(w http.ResponseWriter, notification map[string][]Notifica
|
||||
// CRLF character used for chunked transfer in accordance with HTTP standards.
|
||||
var crlf = []byte("\r\n")
|
||||
|
||||
// sendBucketNotification - writes notification back to client on the response writer
|
||||
// for each notification input, otherwise writes whitespace characters periodically
|
||||
// to keep the connection active. Each notification messages are terminated by CRLF
|
||||
// character. Upon any error received on response writer the for loop exits.
|
||||
func sendBucketNotification(w http.ResponseWriter, arnListenerCh <-chan []NotificationEvent) {
|
||||
var dummyEvents = map[string][]NotificationEvent{"Records": nil}
|
||||
// Continuously write to client either timely empty structures
|
||||
// every 5 seconds, or return back the notifications.
|
||||
// listenChan A `listenChan` provides a data channel to send event
|
||||
// notifications on and `doneCh` to signal that events are no longer
|
||||
// being received. It also sends empty events (whitespace) to keep the
|
||||
// underlying connection alive.
|
||||
type listenChan struct {
|
||||
doneCh chan struct{}
|
||||
dataCh chan []NotificationEvent
|
||||
}
|
||||
|
||||
// newListenChan returns a listenChan with properly initialized
|
||||
// unbuffered channels.
|
||||
func newListenChan() *listenChan {
|
||||
return &listenChan{
|
||||
doneCh: make(chan struct{}),
|
||||
dataCh: make(chan []NotificationEvent),
|
||||
}
|
||||
}
|
||||
|
||||
// sendNotificationEvent sends notification events on the data channel
|
||||
// unless doneCh is not closed
|
||||
func (l *listenChan) sendNotificationEvent(events []NotificationEvent) {
|
||||
select {
|
||||
// Returns immediately if receiver has quit.
|
||||
case <-l.doneCh:
|
||||
// Blocks until receiver is available.
|
||||
case l.dataCh <- events:
|
||||
}
|
||||
}
|
||||
|
||||
// waitForListener writes event notification OR whitespaces on
|
||||
// ResponseWriter until client closes connection
|
||||
func (l *listenChan) waitForListener(w http.ResponseWriter) {
|
||||
|
||||
// Logs errors other than EPIPE and ECONNRESET.
|
||||
// EPIPE and ECONNRESET indicate that the client stopped
|
||||
// listening to notification events.
|
||||
logClientError := func(err error, msg string) {
|
||||
if oe, ok := err.(*net.OpError); ok && (oe.Err == syscall.EPIPE || oe.Err ==
|
||||
syscall.ECONNRESET) {
|
||||
errorIf(err, msg)
|
||||
}
|
||||
}
|
||||
|
||||
emptyEvent := map[string][]NotificationEvent{"Records": nil}
|
||||
defer close(l.doneCh)
|
||||
for {
|
||||
select {
|
||||
case events := <-arnListenerCh:
|
||||
case events := <-l.dataCh:
|
||||
if err := writeNotification(w, map[string][]NotificationEvent{"Records": events}); err != nil {
|
||||
errorIf(err, "Unable to write notification to client.")
|
||||
logClientError(err, "Unable to write notification")
|
||||
return
|
||||
}
|
||||
case <-time.After(globalSNSConnAlive): // Wait for global conn active seconds.
|
||||
if err := writeNotification(w, dummyEvents); err != nil {
|
||||
// FIXME - do not log for all errors.
|
||||
errorIf(err, "Unable to write notification to client.")
|
||||
case <-time.After(globalSNSConnAlive):
|
||||
if err := writeNotification(w, emptyEvent); err != nil {
|
||||
logClientError(err, "Unable to write empty notification")
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -296,6 +332,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
return
|
||||
}
|
||||
|
||||
targetServer := GetLocalPeer(globalEndpoints)
|
||||
accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
|
||||
accountARN := fmt.Sprintf(
|
||||
"%s:%s:%s:%s-%s",
|
||||
@@ -303,8 +340,9 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
serverConfig.GetRegion(),
|
||||
accountID,
|
||||
snsTypeMinio,
|
||||
globalMinioAddr,
|
||||
targetServer,
|
||||
)
|
||||
|
||||
var filterRules []filterRule
|
||||
|
||||
for _, prefix := range prefixes {
|
||||
@@ -338,12 +376,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
},
|
||||
}
|
||||
|
||||
// Setup a listening channel that will receive notifications
|
||||
// from the RPC handler.
|
||||
nEventCh := make(chan []NotificationEvent)
|
||||
defer close(nEventCh)
|
||||
// Setup a listen channel to receive notifications like
|
||||
// s3:ObjectCreated, s3:ObjectDeleted etc.
|
||||
nListenCh := newListenChan()
|
||||
// Add channel for listener events
|
||||
if err = globalEventNotifier.AddListenerChan(accountARN, nEventCh); err != nil {
|
||||
if err = globalEventNotifier.AddListenerChan(accountARN, nListenCh); err != nil {
|
||||
errorIf(err, "Error adding a listener!")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
@@ -353,11 +390,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
defer globalEventNotifier.RemoveListenerChan(accountARN)
|
||||
|
||||
// Update topic config to bucket config and persist - as soon
|
||||
// as this call compelets, events may start appearing in
|
||||
// nEventCh
|
||||
// as this call completes, events may start appearing in
|
||||
// nListenCh
|
||||
lc := listenerConfig{
|
||||
TopicConfig: *topicCfg,
|
||||
TargetServer: globalMinioAddr,
|
||||
TargetServer: targetServer,
|
||||
}
|
||||
|
||||
err = AddBucketListenerConfig(bucket, &lc, objAPI)
|
||||
@@ -370,8 +407,16 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
// Add all common headers.
|
||||
setCommonHeaders(w)
|
||||
|
||||
// Start sending bucket notifications.
|
||||
sendBucketNotification(w, nEventCh)
|
||||
// https://github.com/containous/traefik/issues/560
|
||||
// https://developer.mozilla.org/en-US/docs/Web/API/Server-sent_events/Using_server-sent_events
|
||||
//
|
||||
// Proxies might buffer the connection to avoid this we
|
||||
// need the proper MIME type before writing to client.
|
||||
// This MIME header tells the proxies to avoid buffering
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
|
||||
// Start writing bucket notifications to ResponseWriter.
|
||||
nListenCh.waitForListener(w)
|
||||
}
|
||||
|
||||
// AddBucketListenerConfig - Updates on disk state of listeners, and
|
||||
|
||||
@@ -21,13 +21,19 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// Implement a dummy flush writer.
|
||||
@@ -111,7 +117,47 @@ func TestWriteNotification(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendBucketNotification(t *testing.T) {
|
||||
// testResponseWriter implements `http.ResponseWriter` that buffers
|
||||
// response body in a `bytes.Buffer` and returns error after `failCount`
|
||||
// calls to `Write` method
|
||||
type testResponseWriter struct {
|
||||
mu sync.Mutex
|
||||
failCount int
|
||||
buf *bytes.Buffer
|
||||
m http.Header
|
||||
}
|
||||
|
||||
func newTestResponseWriter(failAt int) *testResponseWriter {
|
||||
return &testResponseWriter{
|
||||
buf: new(bytes.Buffer),
|
||||
m: make(http.Header),
|
||||
failCount: failAt,
|
||||
}
|
||||
}
|
||||
|
||||
func (trw *testResponseWriter) Flush() {
|
||||
}
|
||||
|
||||
func (trw *testResponseWriter) Write(p []byte) (int, error) {
|
||||
trw.mu.Lock()
|
||||
defer trw.mu.Unlock()
|
||||
|
||||
if trw.failCount == 0 {
|
||||
return 0, errors.New("Custom error")
|
||||
}
|
||||
trw.failCount--
|
||||
|
||||
return trw.buf.Write(p)
|
||||
}
|
||||
|
||||
func (trw *testResponseWriter) Header() http.Header {
|
||||
return trw.m
|
||||
}
|
||||
|
||||
func (trw *testResponseWriter) WriteHeader(i int) {
|
||||
}
|
||||
|
||||
func TestListenChan(t *testing.T) {
|
||||
// Initialize a new test config.
|
||||
root, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
@@ -119,18 +165,8 @@ func TestSendBucketNotification(t *testing.T) {
|
||||
}
|
||||
defer os.RemoveAll(root)
|
||||
|
||||
eventCh := make(chan []NotificationEvent)
|
||||
|
||||
// Create a Pipe with FlushWriter on the write-side and bufio.Scanner
|
||||
// on the reader-side to receive notification over the listen channel in a
|
||||
// synchronized manner.
|
||||
pr, pw := io.Pipe()
|
||||
fw := newFlushWriter(pw)
|
||||
scanner := bufio.NewScanner(pr)
|
||||
// Start a go-routine to wait for notification events.
|
||||
go func(listenerCh <-chan []NotificationEvent) {
|
||||
sendBucketNotification(fw, listenerCh)
|
||||
}(eventCh)
|
||||
// Create a listen channel to manage notifications
|
||||
nListenCh := newListenChan()
|
||||
|
||||
// Construct notification events to be passed on the events channel.
|
||||
var events []NotificationEvent
|
||||
@@ -140,37 +176,68 @@ func TestSendBucketNotification(t *testing.T) {
|
||||
ObjectCreatedCopy,
|
||||
ObjectCreatedCompleteMultipartUpload,
|
||||
}
|
||||
|
||||
for _, evType := range evTypes {
|
||||
events = append(events, newNotificationEvent(eventData{
|
||||
Type: evType,
|
||||
}))
|
||||
}
|
||||
// Send notification events to the channel on which sendBucketNotification
|
||||
// is waiting on.
|
||||
eventCh <- events
|
||||
|
||||
// Read from the pipe connected to the ResponseWriter.
|
||||
scanner.Scan()
|
||||
notificationBytes := scanner.Bytes()
|
||||
|
||||
// Close the read-end and send an empty notification event on the channel
|
||||
// to signal sendBucketNotification to terminate.
|
||||
pr.Close()
|
||||
eventCh <- []NotificationEvent{}
|
||||
close(eventCh)
|
||||
|
||||
// Checking if the notification are the same as those sent over the channel.
|
||||
var notifications map[string][]NotificationEvent
|
||||
err = json.Unmarshal(notificationBytes, ¬ifications)
|
||||
if err != nil {
|
||||
t.Fatal("Failed to Unmarshal notification")
|
||||
}
|
||||
records := notifications["Records"]
|
||||
for i, rec := range records {
|
||||
if rec.EventName == evTypes[i].String() {
|
||||
continue
|
||||
// Send notification events one-by-one
|
||||
go func() {
|
||||
for _, event := range events {
|
||||
nListenCh.sendNotificationEvent([]NotificationEvent{event})
|
||||
}
|
||||
t.Errorf("Failed to receive %d event %s", i, evTypes[i].String())
|
||||
}()
|
||||
|
||||
// Create a http.ResponseWriter that fails after len(events)
|
||||
// number of times
|
||||
trw := newTestResponseWriter(len(events))
|
||||
|
||||
// Wait for all (4) notification events to be received
|
||||
nListenCh.waitForListener(trw)
|
||||
|
||||
// Used to read JSON-formatted event stream line-by-line
|
||||
scanner := bufio.NewScanner(trw.buf)
|
||||
var records map[string][]NotificationEvent
|
||||
for i := 0; scanner.Scan(); i++ {
|
||||
err = json.Unmarshal(scanner.Bytes(), &records)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to unmarshal json %v", err)
|
||||
}
|
||||
|
||||
nEvent := records["Records"][0]
|
||||
if nEvent.EventName != evTypes[i].String() {
|
||||
t.Errorf("notification event name mismatch, expected %s but got %s", evTypes[i], nEvent.EventName)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSendNotificationEvent(t *testing.T) {
|
||||
// This test verifies that sendNotificationEvent function
|
||||
// returns once listenChan.doneCh is closed
|
||||
|
||||
l := newListenChan()
|
||||
testCh := make(chan struct{})
|
||||
timeout := 5 * time.Second
|
||||
|
||||
go func() {
|
||||
// Send one empty notification event on listenChan
|
||||
events := []NotificationEvent{NotificationEvent{}}
|
||||
l.sendNotificationEvent(events)
|
||||
testCh <- struct{}{}
|
||||
}()
|
||||
|
||||
// close l.doneCh to signal client exiting from
|
||||
// ListenBucketNotification API call
|
||||
close(l.doneCh)
|
||||
|
||||
select {
|
||||
case <-time.After(timeout):
|
||||
t.Fatalf("sendNotificationEvent didn't return after %v seconds", timeout)
|
||||
case <-testCh:
|
||||
// If we reach this case, sendNotificationEvent
|
||||
// returned on closing l.doneCh
|
||||
}
|
||||
}
|
||||
|
||||
@@ -181,7 +248,7 @@ func TestGetBucketNotificationHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
func testGetBucketNotificationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
// declare sample configs
|
||||
filterRules := []filterRule{
|
||||
{
|
||||
@@ -254,7 +321,7 @@ func TestPutBucketNotificationHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
func testPutBucketNotificationHandler(obj ObjectLayer, instanceType,
|
||||
bucketName string, apiRouter http.Handler, credentials credential,
|
||||
bucketName string, apiRouter http.Handler, credentials auth.Credentials,
|
||||
t *testing.T) {
|
||||
|
||||
// declare sample configs
|
||||
@@ -344,7 +411,7 @@ func TestListenBucketNotificationNilHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
// get random bucket name.
|
||||
randBucket := getRandomBucketName()
|
||||
|
||||
@@ -371,7 +438,7 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
|
||||
}
|
||||
|
||||
func testRemoveNotificationConfig(obj ObjectLayer, instanceType,
|
||||
bucketName string, apiRouter http.Handler, credentials credential,
|
||||
bucketName string, apiRouter http.Handler, credentials auth.Credentials,
|
||||
t *testing.T) {
|
||||
|
||||
invalidBucket := "Invalid\\Bucket"
|
||||
|
||||
@@ -116,9 +116,13 @@ func checkARN(arn, arnType string) APIErrorCode {
|
||||
if len(strs) != 6 {
|
||||
return ErrARNNotification
|
||||
}
|
||||
if serverConfig.GetRegion() != "" {
|
||||
|
||||
// Server region is allowed to be empty by default,
|
||||
// in such a scenario ARN region is not validating
|
||||
// allowing all regions.
|
||||
if sregion := serverConfig.GetRegion(); sregion != "" {
|
||||
region := strs[3]
|
||||
if region != serverConfig.GetRegion() {
|
||||
if region != sregion {
|
||||
return ErrRegionNotification
|
||||
}
|
||||
}
|
||||
@@ -273,9 +277,13 @@ func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
|
||||
if len(strs) != 6 {
|
||||
return
|
||||
}
|
||||
if serverConfig.GetRegion() != "" {
|
||||
|
||||
// Server region is allowed to be empty by default,
|
||||
// in such a scenario ARN region is not validating
|
||||
// allowing all regions.
|
||||
if sregion := serverConfig.GetRegion(); sregion != "" {
|
||||
region := strs[3]
|
||||
if region != serverConfig.GetRegion() {
|
||||
if region != sregion {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
mux "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/wildcard"
|
||||
)
|
||||
|
||||
@@ -36,7 +36,8 @@ const maxAccessPolicySize = 20 * humanize.KiByte
|
||||
|
||||
// Verify if a given action is valid for the url path based on the
|
||||
// existing bucket access policy.
|
||||
func bucketPolicyEvalStatements(action string, resource string, conditions map[string]set.StringSet, statements []policyStatement) bool {
|
||||
func bucketPolicyEvalStatements(action string, resource string, conditions policy.ConditionKeyMap,
|
||||
statements []policy.Statement) bool {
|
||||
for _, statement := range statements {
|
||||
if bucketPolicyMatchStatement(action, resource, conditions, statement) {
|
||||
if statement.Effect == "Allow" {
|
||||
@@ -52,7 +53,8 @@ func bucketPolicyEvalStatements(action string, resource string, conditions map[s
|
||||
}
|
||||
|
||||
// Verify if action, resource and conditions match input policy statement.
|
||||
func bucketPolicyMatchStatement(action string, resource string, conditions map[string]set.StringSet, statement policyStatement) bool {
|
||||
func bucketPolicyMatchStatement(action string, resource string, conditions policy.ConditionKeyMap,
|
||||
statement policy.Statement) bool {
|
||||
// Verify if action, resource and condition match in given statement.
|
||||
return (bucketPolicyActionMatch(action, statement) &&
|
||||
bucketPolicyResourceMatch(resource, statement) &&
|
||||
@@ -60,7 +62,7 @@ func bucketPolicyMatchStatement(action string, resource string, conditions map[s
|
||||
}
|
||||
|
||||
// Verify if given action matches with policy statement.
|
||||
func bucketPolicyActionMatch(action string, statement policyStatement) bool {
|
||||
func bucketPolicyActionMatch(action string, statement policy.Statement) bool {
|
||||
return !statement.Actions.FuncMatch(actionMatch, action).IsEmpty()
|
||||
}
|
||||
|
||||
@@ -95,7 +97,7 @@ func isIPInCIDR(cidr, ip string) bool {
|
||||
}
|
||||
|
||||
// Verify if given resource matches with policy statement.
|
||||
func bucketPolicyResourceMatch(resource string, statement policyStatement) bool {
|
||||
func bucketPolicyResourceMatch(resource string, statement policy.Statement) bool {
|
||||
// the resource rule for object could contain "*" wild card.
|
||||
// the requested object can be given access based on the already set bucket policy if
|
||||
// the match is successful.
|
||||
@@ -104,7 +106,7 @@ func bucketPolicyResourceMatch(resource string, statement policyStatement) bool
|
||||
}
|
||||
|
||||
// Verify if given condition matches with policy statement.
|
||||
func bucketPolicyConditionMatch(conditions map[string]set.StringSet, statement policyStatement) bool {
|
||||
func bucketPolicyConditionMatch(conditions policy.ConditionKeyMap, statement policy.Statement) bool {
|
||||
// Supports following conditions.
|
||||
// - StringEquals
|
||||
// - StringNotEquals
|
||||
@@ -300,13 +302,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
|
||||
// Delete bucket access policy, by passing an empty policy
|
||||
// struct.
|
||||
if err := persistAndNotifyBucketPolicyChange(bucket, policyChange{true, nil}, objAPI); err != nil {
|
||||
switch err.(type) {
|
||||
case BucketPolicyNotFound:
|
||||
writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
|
||||
default:
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
}
|
||||
err = persistAndNotifyBucketPolicyChange(bucket, policyChange{
|
||||
true, policy.BucketAccessPolicy{},
|
||||
}, objAPI)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -345,15 +345,17 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
policy, err := readBucketPolicy(bucket, objAPI)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
switch err.(type) {
|
||||
case BucketPolicyNotFound:
|
||||
writeErrorResponse(w, ErrNoSuchBucketPolicy, r.URL)
|
||||
default:
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
}
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
policyBytes, err := json.Marshal(&policy)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to marshal bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Write to client.
|
||||
fmt.Fprint(w, policy)
|
||||
w.Write(policyBytes)
|
||||
}
|
||||
|
||||
@@ -25,15 +25,17 @@ import (
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// Tests validate Bucket policy resource matcher.
|
||||
func TestBucketPolicyResourceMatch(t *testing.T) {
|
||||
|
||||
// generates statement with given resource..
|
||||
generateStatement := func(resource string) policyStatement {
|
||||
statement := policyStatement{}
|
||||
generateStatement := func(resource string) policy.Statement {
|
||||
statement := policy.Statement{}
|
||||
statement.Resources = set.CreateStringSet([]string{resource}...)
|
||||
return statement
|
||||
}
|
||||
@@ -45,7 +47,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
resourceToMatch string
|
||||
statement policyStatement
|
||||
statement policy.Statement
|
||||
expectedResourceMatch bool
|
||||
}{
|
||||
// Test case 1-4.
|
||||
@@ -85,7 +87,7 @@ func TestBucketPolicyResourceMatch(t *testing.T) {
|
||||
}
|
||||
|
||||
// TestBucketPolicyActionMatch - Test validates whether given action on the
|
||||
// bucket/object matches the allowed actions in policyStatement.
|
||||
// bucket/object matches the allowed actions in policy.Statement.
|
||||
// This test preserves the allowed actions for all 3 sets of policies, that is read-write,read-only, write-only.
|
||||
// The intention of the test is to catch any changes made to allowed action for on eof the above 3 major policy groups mentioned.
|
||||
func TestBucketPolicyActionMatch(t *testing.T) {
|
||||
@@ -94,7 +96,7 @@ func TestBucketPolicyActionMatch(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
action string
|
||||
statement policyStatement
|
||||
statement policy.Statement
|
||||
expectedResult bool
|
||||
}{
|
||||
// s3:GetBucketLocation is the action necessary to be present in the bucket policy to allow
|
||||
@@ -247,7 +249,7 @@ func TestPutBucketPolicyHandler(t *testing.T) {
|
||||
|
||||
// testPutBucketPolicyHandler - Test for Bucket policy end point.
|
||||
func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
initBucketPolicies(obj)
|
||||
|
||||
bucketName1 := fmt.Sprintf("%s-1", bucketName)
|
||||
@@ -454,7 +456,7 @@ func TestGetBucketPolicyHandler(t *testing.T) {
|
||||
|
||||
// testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket.
|
||||
func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
// initialize bucket policy.
|
||||
initBucketPolicies(obj)
|
||||
|
||||
@@ -643,7 +645,7 @@ func TestDeleteBucketPolicyHandler(t *testing.T) {
|
||||
|
||||
// testDeleteBucketPolicyHandler - Test for Delete bucket policy end point.
|
||||
func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
// initialize bucket policy.
|
||||
initBucketPolicies(obj)
|
||||
|
||||
@@ -843,28 +845,28 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
|
||||
|
||||
// TestBucketPolicyConditionMatch - Tests to validate whether bucket policy conditions match.
|
||||
func TestBucketPolicyConditionMatch(t *testing.T) {
|
||||
// obtain the inner map[string]set.StringSet for policyStatement.Conditions.
|
||||
// obtain the inner map[string]set.StringSet for policy.Statement.Conditions.
|
||||
getInnerMap := func(key2, value string) map[string]set.StringSet {
|
||||
innerMap := make(map[string]set.StringSet)
|
||||
innerMap[key2] = set.CreateStringSet(value)
|
||||
return innerMap
|
||||
}
|
||||
|
||||
// obtain policyStatement with Conditions set.
|
||||
getStatementWithCondition := func(key1, key2, value string) policyStatement {
|
||||
// obtain policy.Statement with Conditions set.
|
||||
getStatementWithCondition := func(key1, key2, value string) policy.Statement {
|
||||
innerMap := getInnerMap(key2, value)
|
||||
// to set policyStatment.Conditions .
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions[key1] = innerMap
|
||||
// new policy statement.
|
||||
statement := policyStatement{}
|
||||
statement := policy.Statement{}
|
||||
// set the condition.
|
||||
statement.Conditions = conditions
|
||||
return statement
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
statementCondition policyStatement
|
||||
statementCondition policy.Statement
|
||||
condition map[string]set.StringSet
|
||||
|
||||
expectedMatch bool
|
||||
|
||||
+32
-95
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -26,12 +26,16 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
var conditionKeyActionMap = map[string]set.StringSet{
|
||||
"s3:prefix": set.CreateStringSet("s3:ListBucket"),
|
||||
"s3:max-keys": set.CreateStringSet("s3:ListBucket"),
|
||||
var emptyBucketPolicy = policy.BucketAccessPolicy{}
|
||||
|
||||
var conditionKeyActionMap = policy.ConditionKeyMap{
|
||||
"s3:prefix": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads"),
|
||||
"s3:max-keys": set.CreateStringSet("s3:ListBucket", "s3:ListBucketMultipartUploads",
|
||||
"s3:ListMultipartUploadParts"),
|
||||
}
|
||||
|
||||
// supportedActionMap - lists all the actions supported by minio.
|
||||
@@ -49,41 +53,16 @@ var supportedConditionsKey = set.CreateStringSet("s3:prefix", "s3:max-keys", "aw
|
||||
// supportedEffectMap - supported effects.
|
||||
var supportedEffectMap = set.CreateStringSet("Allow", "Deny")
|
||||
|
||||
// Statement - minio policy statement
|
||||
type policyStatement struct {
|
||||
Actions set.StringSet `json:"Action"`
|
||||
Conditions map[string]map[string]set.StringSet `json:"Condition,omitempty"`
|
||||
Effect string
|
||||
Principal interface{} `json:"Principal"`
|
||||
Resources set.StringSet `json:"Resource"`
|
||||
Sid string
|
||||
}
|
||||
|
||||
// bucketPolicy - collection of various bucket policy statements.
|
||||
type bucketPolicy struct {
|
||||
Version string // date in YYYY-MM-DD format
|
||||
Statements []policyStatement `json:"Statement"`
|
||||
}
|
||||
|
||||
// Stringer implementation for the bucket policies.
|
||||
func (b bucketPolicy) String() string {
|
||||
bbytes, err := json.Marshal(&b)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to marshal bucket policy into JSON %#v", b)
|
||||
return ""
|
||||
}
|
||||
return string(bbytes)
|
||||
}
|
||||
|
||||
// isValidActions - are actions valid.
|
||||
func isValidActions(actions set.StringSet) (err error) {
|
||||
// Statement actions cannot be empty.
|
||||
if len(actions) == 0 {
|
||||
if actions.IsEmpty() {
|
||||
err = errors.New("Action list cannot be empty")
|
||||
return err
|
||||
}
|
||||
if unsupportedActions := actions.Difference(supportedActionMap); !unsupportedActions.IsEmpty() {
|
||||
err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document", unsupportedActions)
|
||||
err = fmt.Errorf("Unsupported actions found: ‘%#v’, please validate your policy document",
|
||||
unsupportedActions)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -106,7 +85,7 @@ func isValidEffect(effect string) (err error) {
|
||||
// isValidResources - are valid resources.
|
||||
func isValidResources(resources set.StringSet) (err error) {
|
||||
// Statement resources cannot be empty.
|
||||
if len(resources) == 0 {
|
||||
if resources.IsEmpty() {
|
||||
err = errors.New("Resource list cannot be empty")
|
||||
return err
|
||||
}
|
||||
@@ -124,60 +103,17 @@ func isValidResources(resources set.StringSet) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parse principals parses a incoming json. Handles cases for
|
||||
// these three combinations.
|
||||
// - "Principal": "*",
|
||||
// - "Principal": { "AWS" : "*" }
|
||||
// - "Principal": { "AWS" : [ "*" ]}
|
||||
func parsePrincipals(principal interface{}) set.StringSet {
|
||||
principals, ok := principal.(map[string]interface{})
|
||||
if !ok {
|
||||
var principalStr string
|
||||
principalStr, ok = principal.(string)
|
||||
if ok {
|
||||
return set.CreateStringSet(principalStr)
|
||||
}
|
||||
} // else {
|
||||
var principalStrs []string
|
||||
for _, p := range principals {
|
||||
principalStr, isStr := p.(string)
|
||||
if !isStr {
|
||||
principalsAdd, isInterface := p.([]interface{})
|
||||
if !isInterface {
|
||||
principalStrsAddr, isStrs := p.([]string)
|
||||
if !isStrs {
|
||||
continue
|
||||
}
|
||||
principalStrs = append(principalStrs, principalStrsAddr...)
|
||||
} else {
|
||||
for _, pa := range principalsAdd {
|
||||
var pstr string
|
||||
pstr, isStr = pa.(string)
|
||||
if !isStr {
|
||||
continue
|
||||
}
|
||||
principalStrs = append(principalStrs, pstr)
|
||||
}
|
||||
}
|
||||
continue
|
||||
} // else {
|
||||
principalStrs = append(principalStrs, principalStr)
|
||||
}
|
||||
return set.CreateStringSet(principalStrs...)
|
||||
}
|
||||
|
||||
// isValidPrincipals - are valid principals.
|
||||
func isValidPrincipals(principal interface{}) (err error) {
|
||||
principals := parsePrincipals(principal)
|
||||
// Statement principal should have a value.
|
||||
if len(principals) == 0 {
|
||||
err = errors.New("Principal cannot be empty")
|
||||
return err
|
||||
func isValidPrincipals(principal policy.User) (err error) {
|
||||
if principal.AWS.IsEmpty() {
|
||||
return errors.New("Principal cannot be empty")
|
||||
}
|
||||
if unsuppPrincipals := principals.Difference(set.CreateStringSet([]string{"*"}...)); !unsuppPrincipals.IsEmpty() {
|
||||
if diff := principal.AWS.Difference(set.CreateStringSet("*")); !diff.IsEmpty() {
|
||||
// Minio does not support or implement IAM, "*" is the only valid value.
|
||||
// Amazon s3 doc on principals: http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
|
||||
err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document", unsuppPrincipals)
|
||||
// Amazon s3 doc on principal:
|
||||
// http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements.html#Principal
|
||||
err = fmt.Errorf("Unsupported principals found: ‘%#v’, please validate your policy document",
|
||||
diff)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
@@ -185,7 +121,7 @@ func isValidPrincipals(principal interface{}) (err error) {
|
||||
|
||||
// isValidConditions - returns nil if the given conditions valid and
|
||||
// corresponding error otherwise.
|
||||
func isValidConditions(actions set.StringSet, conditions map[string]map[string]set.StringSet) (err error) {
|
||||
func isValidConditions(actions set.StringSet, conditions policy.ConditionMap) (err error) {
|
||||
// Verify conditions should be valid. Validate if only
|
||||
// supported condition keys are present and return error
|
||||
// otherwise.
|
||||
@@ -239,7 +175,7 @@ func resourcePrefix(resource string) string {
|
||||
// checkBucketPolicyResources validates Resources in unmarshalled bucket policy structure.
|
||||
// - Resources are validated against the given set of Actions.
|
||||
// -
|
||||
func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIErrorCode {
|
||||
func checkBucketPolicyResources(bucket string, bucketPolicy policy.BucketAccessPolicy) APIErrorCode {
|
||||
// Validate statements for special actions and collect resources
|
||||
// for others to validate nesting.
|
||||
var resourceMap = set.NewStringSet()
|
||||
@@ -294,27 +230,27 @@ func checkBucketPolicyResources(bucket string, bucketPolicy *bucketPolicy) APIEr
|
||||
|
||||
// parseBucketPolicy - parses and validates if bucket policy is of
|
||||
// proper JSON and follows allowed restrictions with policy standards.
|
||||
func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err error) {
|
||||
func parseBucketPolicy(bucketPolicyReader io.Reader, bktPolicy *policy.BucketAccessPolicy) (err error) {
|
||||
// Parse bucket policy reader.
|
||||
decoder := json.NewDecoder(bucketPolicyReader)
|
||||
if err = decoder.Decode(&policy); err != nil {
|
||||
if err = decoder.Decode(bktPolicy); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Policy version cannot be empty.
|
||||
if len(policy.Version) == 0 {
|
||||
if len(bktPolicy.Version) == 0 {
|
||||
err = errors.New("Policy version cannot be empty")
|
||||
return err
|
||||
}
|
||||
|
||||
// Policy statements cannot be empty.
|
||||
if len(policy.Statements) == 0 {
|
||||
if len(bktPolicy.Statements) == 0 {
|
||||
err = errors.New("Policy statement cannot be empty")
|
||||
return err
|
||||
}
|
||||
|
||||
// Loop through all policy statements and validate entries.
|
||||
for _, statement := range policy.Statements {
|
||||
for _, statement := range bktPolicy.Statements {
|
||||
// Statement effect should be valid.
|
||||
if err := isValidEffect(statement.Effect); err != nil {
|
||||
return err
|
||||
@@ -339,19 +275,20 @@ func parseBucketPolicy(bucketPolicyReader io.Reader, policy *bucketPolicy) (err
|
||||
|
||||
// Separate deny and allow statements, so that we can apply deny
|
||||
// statements in the beginning followed by Allow statements.
|
||||
var denyStatements []policyStatement
|
||||
var allowStatements []policyStatement
|
||||
for _, statement := range policy.Statements {
|
||||
var denyStatements []policy.Statement
|
||||
var allowStatements []policy.Statement
|
||||
for _, statement := range bktPolicy.Statements {
|
||||
if statement.Effect == "Deny" {
|
||||
denyStatements = append(denyStatements, statement)
|
||||
continue
|
||||
}
|
||||
|
||||
// else if statement.Effect == "Allow"
|
||||
allowStatements = append(allowStatements, statement)
|
||||
}
|
||||
|
||||
// Deny statements are enforced first once matched.
|
||||
policy.Statements = append(denyStatements, allowStatements...)
|
||||
bktPolicy.Statements = append(denyStatements, allowStatements...)
|
||||
|
||||
// Return successfully parsed policy structure.
|
||||
return nil
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
@@ -74,11 +75,11 @@ var (
|
||||
)
|
||||
|
||||
// Obtain bucket statement for read-write bucketPolicy.
|
||||
func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatement {
|
||||
objectResourceStatement := policyStatement{}
|
||||
func getReadWriteObjectStatement(bucketName, objectPrefix string) policy.Statement {
|
||||
objectResourceStatement := policy.Statement{}
|
||||
objectResourceStatement.Effect = "Allow"
|
||||
objectResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
objectResourceStatement.Principal = policy.User{
|
||||
AWS: set.StringSet{"*": struct{}{}},
|
||||
}
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Actions = set.CreateStringSet(readWriteObjectActions...)
|
||||
@@ -86,11 +87,11 @@ func getReadWriteObjectStatement(bucketName, objectPrefix string) policyStatemen
|
||||
}
|
||||
|
||||
// Obtain object statement for read-write bucketPolicy.
|
||||
func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatement {
|
||||
bucketResourceStatement := policyStatement{}
|
||||
func getReadWriteBucketStatement(bucketName, objectPrefix string) policy.Statement {
|
||||
bucketResourceStatement := policy.Statement{}
|
||||
bucketResourceStatement.Effect = "Allow"
|
||||
bucketResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
bucketResourceStatement.Principal = policy.User{
|
||||
AWS: set.StringSet{"*": struct{}{}},
|
||||
}
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
|
||||
bucketResourceStatement.Actions = set.CreateStringSet(readWriteBucketActions...)
|
||||
@@ -98,19 +99,19 @@ func getReadWriteBucketStatement(bucketName, objectPrefix string) policyStatemen
|
||||
}
|
||||
|
||||
// Obtain statements for read-write bucketPolicy.
|
||||
func getReadWriteStatement(bucketName, objectPrefix string) []policyStatement {
|
||||
statements := []policyStatement{}
|
||||
func getReadWriteStatement(bucketName, objectPrefix string) []policy.Statement {
|
||||
statements := []policy.Statement{}
|
||||
// Save the read write policy.
|
||||
statements = append(statements, getReadWriteBucketStatement(bucketName, objectPrefix), getReadWriteObjectStatement(bucketName, objectPrefix))
|
||||
return statements
|
||||
}
|
||||
|
||||
// Obtain bucket statement for read only bucketPolicy.
|
||||
func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
|
||||
bucketResourceStatement := policyStatement{}
|
||||
func getReadOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
|
||||
bucketResourceStatement := policy.Statement{}
|
||||
bucketResourceStatement.Effect = "Allow"
|
||||
bucketResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
bucketResourceStatement.Principal = policy.User{
|
||||
AWS: set.StringSet{"*": struct{}{}},
|
||||
}
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
|
||||
bucketResourceStatement.Actions = set.CreateStringSet(readOnlyBucketActions...)
|
||||
@@ -118,11 +119,11 @@ func getReadOnlyBucketStatement(bucketName, objectPrefix string) policyStatement
|
||||
}
|
||||
|
||||
// Obtain object statement for read only bucketPolicy.
|
||||
func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
|
||||
objectResourceStatement := policyStatement{}
|
||||
func getReadOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
|
||||
objectResourceStatement := policy.Statement{}
|
||||
objectResourceStatement.Effect = "Allow"
|
||||
objectResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
objectResourceStatement.Principal = policy.User{
|
||||
AWS: set.StringSet{"*": struct{}{}},
|
||||
}
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Actions = set.CreateStringSet(readOnlyObjectActions...)
|
||||
@@ -130,20 +131,20 @@ func getReadOnlyObjectStatement(bucketName, objectPrefix string) policyStatement
|
||||
}
|
||||
|
||||
// Obtain statements for read only bucketPolicy.
|
||||
func getReadOnlyStatement(bucketName, objectPrefix string) []policyStatement {
|
||||
statements := []policyStatement{}
|
||||
func getReadOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
|
||||
statements := []policy.Statement{}
|
||||
// Save the read only policy.
|
||||
statements = append(statements, getReadOnlyBucketStatement(bucketName, objectPrefix), getReadOnlyObjectStatement(bucketName, objectPrefix))
|
||||
return statements
|
||||
}
|
||||
|
||||
// Obtain bucket statements for write only bucketPolicy.
|
||||
func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatement {
|
||||
func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policy.Statement {
|
||||
|
||||
bucketResourceStatement := policyStatement{}
|
||||
bucketResourceStatement := policy.Statement{}
|
||||
bucketResourceStatement.Effect = "Allow"
|
||||
bucketResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
bucketResourceStatement.Principal = policy.User{
|
||||
AWS: set.StringSet{"*": struct{}{}},
|
||||
}
|
||||
bucketResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName)}...)
|
||||
bucketResourceStatement.Actions = set.CreateStringSet(writeOnlyBucketActions...)
|
||||
@@ -151,11 +152,11 @@ func getWriteOnlyBucketStatement(bucketName, objectPrefix string) policyStatemen
|
||||
}
|
||||
|
||||
// Obtain object statements for write only bucketPolicy.
|
||||
func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatement {
|
||||
objectResourceStatement := policyStatement{}
|
||||
func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policy.Statement {
|
||||
objectResourceStatement := policy.Statement{}
|
||||
objectResourceStatement.Effect = "Allow"
|
||||
objectResourceStatement.Principal = map[string]interface{}{
|
||||
"AWS": "*",
|
||||
objectResourceStatement.Principal = policy.User{
|
||||
AWS: set.StringSet{"*": struct{}{}},
|
||||
}
|
||||
objectResourceStatement.Resources = set.CreateStringSet([]string{fmt.Sprintf("%s%s", bucketARNPrefix, bucketName+"/"+objectPrefix+"*")}...)
|
||||
objectResourceStatement.Actions = set.CreateStringSet(writeOnlyObjectActions...)
|
||||
@@ -163,8 +164,8 @@ func getWriteOnlyObjectStatement(bucketName, objectPrefix string) policyStatemen
|
||||
}
|
||||
|
||||
// Obtain statements for write only bucketPolicy.
|
||||
func getWriteOnlyStatement(bucketName, objectPrefix string) []policyStatement {
|
||||
statements := []policyStatement{}
|
||||
func getWriteOnlyStatement(bucketName, objectPrefix string) []policy.Statement {
|
||||
statements := []policy.Statement{}
|
||||
// Write only policy.
|
||||
// Save the write only policy.
|
||||
statements = append(statements, getWriteOnlyBucketStatement(bucketName, objectPrefix), getWriteOnlyBucketStatement(bucketName, objectPrefix))
|
||||
@@ -325,9 +326,10 @@ func TestIsValidPrincipals(t *testing.T) {
|
||||
{[]string{"*"}, nil, true},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
err := isValidPrincipals(map[string]interface{}{
|
||||
"AWS": testCase.principals,
|
||||
})
|
||||
u := policy.User{
|
||||
AWS: set.CreateStringSet(testCase.principals...),
|
||||
}
|
||||
err := isValidPrincipals(u)
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
|
||||
}
|
||||
@@ -343,86 +345,86 @@ func TestIsValidPrincipals(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// getEmptyConditionKeyMap - returns a function that generates a
|
||||
// getEmptyConditionMap - returns a function that generates a
|
||||
// condition key map for a given key.
|
||||
func getEmptyConditionKeyMap(conditionKey string) func() map[string]map[string]set.StringSet {
|
||||
emptyConditonGenerator := func() map[string]map[string]set.StringSet {
|
||||
emptyMap := make(map[string]set.StringSet)
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
func getEmptyConditionMap(conditionKey string) func() policy.ConditionMap {
|
||||
emptyConditonGenerator := func() policy.ConditionMap {
|
||||
emptyMap := make(policy.ConditionKeyMap)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions[conditionKey] = emptyMap
|
||||
return conditions
|
||||
}
|
||||
return emptyConditonGenerator
|
||||
}
|
||||
|
||||
// Tests validate policyStatement condition validator.
|
||||
// Tests validate policy.Statement condition validator.
|
||||
func TestIsValidConditions(t *testing.T) {
|
||||
// returns empty conditions map.
|
||||
setEmptyConditions := func() map[string]map[string]set.StringSet {
|
||||
return make(map[string]map[string]set.StringSet)
|
||||
setEmptyConditions := func() policy.ConditionMap {
|
||||
return make(policy.ConditionMap)
|
||||
}
|
||||
|
||||
// returns map with the "StringEquals" set to empty map.
|
||||
setEmptyStringEquals := getEmptyConditionKeyMap("StringEquals")
|
||||
setEmptyStringEquals := getEmptyConditionMap("StringEquals")
|
||||
|
||||
// returns map with the "StringNotEquals" set to empty map.
|
||||
setEmptyStringNotEquals := getEmptyConditionKeyMap("StringNotEquals")
|
||||
setEmptyStringNotEquals := getEmptyConditionMap("StringNotEquals")
|
||||
|
||||
// returns map with the "StringLike" set to empty map.
|
||||
setEmptyStringLike := getEmptyConditionKeyMap("StringLike")
|
||||
setEmptyStringLike := getEmptyConditionMap("StringLike")
|
||||
|
||||
// returns map with the "StringNotLike" set to empty map.
|
||||
setEmptyStringNotLike := getEmptyConditionKeyMap("StringNotLike")
|
||||
setEmptyStringNotLike := getEmptyConditionMap("StringNotLike")
|
||||
|
||||
// returns map with the "IpAddress" set to empty map.
|
||||
setEmptyIPAddress := getEmptyConditionKeyMap("IpAddress")
|
||||
setEmptyIPAddress := getEmptyConditionMap("IpAddress")
|
||||
|
||||
// returns map with "NotIpAddress" set to empty map.
|
||||
setEmptyNotIPAddress := getEmptyConditionKeyMap("NotIpAddress")
|
||||
setEmptyNotIPAddress := getEmptyConditionMap("NotIpAddress")
|
||||
|
||||
// Generate conditions.
|
||||
generateConditions := func(key1, key2, value string) map[string]map[string]set.StringSet {
|
||||
innerMap := make(map[string]set.StringSet)
|
||||
generateConditions := func(key1, key2, value string) policy.ConditionMap {
|
||||
innerMap := make(policy.ConditionKeyMap)
|
||||
innerMap[key2] = set.CreateStringSet(value)
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions[key1] = innerMap
|
||||
return conditions
|
||||
}
|
||||
|
||||
// generate ambigious conditions.
|
||||
generateAmbigiousConditions := func() map[string]map[string]set.StringSet {
|
||||
prefixMap := make(map[string]set.StringSet)
|
||||
generateAmbigiousConditions := func() policy.ConditionMap {
|
||||
prefixMap := make(policy.ConditionKeyMap)
|
||||
prefixMap["s3:prefix"] = set.CreateStringSet("Asia/")
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions["StringEquals"] = prefixMap
|
||||
conditions["StringNotEquals"] = prefixMap
|
||||
return conditions
|
||||
}
|
||||
|
||||
// generate valid and non valid type in the condition map.
|
||||
generateValidInvalidConditions := func() map[string]map[string]set.StringSet {
|
||||
innerMap := make(map[string]set.StringSet)
|
||||
generateValidInvalidConditions := func() policy.ConditionMap {
|
||||
innerMap := make(policy.ConditionKeyMap)
|
||||
innerMap["s3:prefix"] = set.CreateStringSet("Asia/")
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions["StringEquals"] = innerMap
|
||||
conditions["InvalidType"] = innerMap
|
||||
return conditions
|
||||
}
|
||||
|
||||
// generate valid and invalid keys for valid types in the same condition map.
|
||||
generateValidInvalidConditionKeys := func() map[string]map[string]set.StringSet {
|
||||
innerMapValid := make(map[string]set.StringSet)
|
||||
generateValidInvalidConditionKeys := func() policy.ConditionMap {
|
||||
innerMapValid := make(policy.ConditionKeyMap)
|
||||
innerMapValid["s3:prefix"] = set.CreateStringSet("Asia/")
|
||||
innerMapInValid := make(map[string]set.StringSet)
|
||||
innerMapInValid["s3:invalid"] = set.CreateStringSet("Asia/")
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions["StringEquals"] = innerMapValid
|
||||
conditions["StringEquals"] = innerMapInValid
|
||||
return conditions
|
||||
}
|
||||
|
||||
// List of Conditions used for test cases.
|
||||
testConditions := []map[string]map[string]set.StringSet{
|
||||
testConditions := []policy.ConditionMap{
|
||||
generateConditions("StringValues", "s3:max-keys", "100"),
|
||||
generateConditions("StringEquals", "s3:Object", "100"),
|
||||
generateAmbigiousConditions(),
|
||||
@@ -447,7 +449,7 @@ func TestIsValidConditions(t *testing.T) {
|
||||
"please validate your policy document", "s3:max-keys", getObjectActionSet)
|
||||
testCases := []struct {
|
||||
inputActions set.StringSet
|
||||
inputCondition map[string]map[string]set.StringSet
|
||||
inputCondition policy.ConditionMap
|
||||
// expected result.
|
||||
expectedErr error
|
||||
// flag indicating whether test should pass.
|
||||
@@ -518,26 +520,26 @@ func TestIsValidConditions(t *testing.T) {
|
||||
// Tests validate Policy Action and Resource fields.
|
||||
func TestCheckbucketPolicyResources(t *testing.T) {
|
||||
// constructing policy statement without invalidPrefixActions (check bucket-policy-parser.go).
|
||||
setValidPrefixActions := func(statements []policyStatement) []policyStatement {
|
||||
setValidPrefixActions := func(statements []policy.Statement) []policy.Statement {
|
||||
statements[0].Actions = set.CreateStringSet([]string{"s3:DeleteObject", "s3:PutObject"}...)
|
||||
return statements
|
||||
}
|
||||
// contracting policy statement with recursive resources.
|
||||
// should result in ErrMalformedPolicy
|
||||
setRecurseResource := func(statements []policyStatement) []policyStatement {
|
||||
setRecurseResource := func(statements []policy.Statement) []policy.Statement {
|
||||
statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/Asia/*", "arn:aws:s3:::minio-bucket/Asia/India/*"}...)
|
||||
return statements
|
||||
}
|
||||
|
||||
// constructing policy statement with lexically close characters.
|
||||
// should not result in ErrMalformedPolicy
|
||||
setResourceLexical := func(statements []policyStatement) []policyStatement {
|
||||
setResourceLexical := func(statements []policy.Statement) []policy.Statement {
|
||||
statements[0].Resources = set.CreateStringSet([]string{"arn:aws:s3:::minio-bucket/op*", "arn:aws:s3:::minio-bucket/oo*"}...)
|
||||
return statements
|
||||
}
|
||||
|
||||
// List of bucketPolicy used for tests.
|
||||
bucketAccessPolicies := []bucketPolicy{
|
||||
bucketAccessPolicies := []policy.BucketAccessPolicy{
|
||||
// bucketPolicy - 1.
|
||||
// Contains valid read only policy statement.
|
||||
{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
|
||||
@@ -568,7 +570,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
inputPolicy bucketPolicy
|
||||
inputPolicy policy.BucketAccessPolicy
|
||||
// expected results.
|
||||
apiErrCode APIErrorCode
|
||||
// Flag indicating whether the test should pass.
|
||||
@@ -599,7 +601,7 @@ func TestCheckbucketPolicyResources(t *testing.T) {
|
||||
{bucketAccessPolicies[6], ErrNone, true},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
apiErrCode := checkBucketPolicyResources("minio-bucket", &testCase.inputPolicy)
|
||||
apiErrCode := checkBucketPolicyResources("minio-bucket", testCase.inputPolicy)
|
||||
if apiErrCode != ErrNone && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass, but failed with Errocode %v", i+1, apiErrCode)
|
||||
}
|
||||
@@ -618,39 +620,39 @@ func TestCheckbucketPolicyResources(t *testing.T) {
|
||||
// Tests validate parsing of BucketAccessPolicy.
|
||||
func TestParseBucketPolicy(t *testing.T) {
|
||||
// set Unsupported Actions.
|
||||
setUnsupportedActions := func(statements []policyStatement) []policyStatement {
|
||||
setUnsupportedActions := func(statements []policy.Statement) []policy.Statement {
|
||||
// "s3:DeleteEverything"" is an Unsupported Action.
|
||||
statements[0].Actions = set.CreateStringSet([]string{"s3:GetObject", "s3:ListBucket", "s3:PutObject", "s3:DeleteEverything"}...)
|
||||
return statements
|
||||
}
|
||||
// set unsupported Effect.
|
||||
setUnsupportedEffect := func(statements []policyStatement) []policyStatement {
|
||||
setUnsupportedEffect := func(statements []policy.Statement) []policy.Statement {
|
||||
// Effect "Don't allow" is Unsupported.
|
||||
statements[0].Effect = "DontAllow"
|
||||
return statements
|
||||
}
|
||||
// set unsupported principals.
|
||||
setUnsupportedPrincipals := func(statements []policyStatement) []policyStatement {
|
||||
setUnsupportedPrincipals := func(statements []policy.Statement) []policy.Statement {
|
||||
// "User1111"" is an Unsupported Principal.
|
||||
statements[0].Principal = map[string]interface{}{
|
||||
"AWS": []string{"*", "User1111"},
|
||||
statements[0].Principal = policy.User{
|
||||
AWS: set.CreateStringSet([]string{"*", "User1111"}...),
|
||||
}
|
||||
return statements
|
||||
}
|
||||
// set unsupported Resources.
|
||||
setUnsupportedResources := func(statements []policyStatement) []policyStatement {
|
||||
setUnsupportedResources := func(statements []policy.Statement) []policy.Statement {
|
||||
// "s3:DeleteEverything"" is an Unsupported Action.
|
||||
statements[0].Resources = set.CreateStringSet([]string{"my-resource"}...)
|
||||
return statements
|
||||
}
|
||||
// List of bucketPolicy used for test cases.
|
||||
bucketAccesPolicies := []bucketPolicy{
|
||||
bucketAccesPolicies := []policy.BucketAccessPolicy{
|
||||
// bucketPolicy - 0.
|
||||
// bucketPolicy statement empty.
|
||||
{Version: "1.0"},
|
||||
// bucketPolicy - 1.
|
||||
// bucketPolicy version empty.
|
||||
{Version: "", Statements: []policyStatement{}},
|
||||
{Version: "", Statements: []policy.Statement{}},
|
||||
// bucketPolicy - 2.
|
||||
// Readonly bucketPolicy.
|
||||
{Version: "1.0", Statements: getReadOnlyStatement("minio-bucket", "")},
|
||||
@@ -675,19 +677,19 @@ func TestParseBucketPolicy(t *testing.T) {
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
inputPolicy bucketPolicy
|
||||
inputPolicy policy.BucketAccessPolicy
|
||||
// expected results.
|
||||
expectedPolicy bucketPolicy
|
||||
expectedPolicy policy.BucketAccessPolicy
|
||||
err error
|
||||
// Flag indicating whether the test should pass.
|
||||
shouldPass bool
|
||||
}{
|
||||
// Test case - 1.
|
||||
// bucketPolicy statement empty.
|
||||
{bucketAccesPolicies[0], bucketPolicy{}, errors.New("Policy statement cannot be empty"), false},
|
||||
{bucketAccesPolicies[0], policy.BucketAccessPolicy{}, errors.New("Policy statement cannot be empty"), false},
|
||||
// Test case - 2.
|
||||
// bucketPolicy version empty.
|
||||
{bucketAccesPolicies[1], bucketPolicy{}, errors.New("Policy version cannot be empty"), false},
|
||||
{bucketAccesPolicies[1], policy.BucketAccessPolicy{}, errors.New("Policy version cannot be empty"), false},
|
||||
// Test case - 3.
|
||||
// Readonly bucketPolicy.
|
||||
{bucketAccesPolicies[2], bucketAccesPolicies[2], nil, true},
|
||||
@@ -718,8 +720,8 @@ func TestParseBucketPolicy(t *testing.T) {
|
||||
t.Fatalf("Test %d: Couldn't Marshal bucket policy %s", i+1, err)
|
||||
}
|
||||
|
||||
var actualAccessPolicy = &bucketPolicy{}
|
||||
err = parseBucketPolicy(&buffer, actualAccessPolicy)
|
||||
var actualAccessPolicy = policy.BucketAccessPolicy{}
|
||||
err = parseBucketPolicy(&buffer, &actualAccessPolicy)
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err.Error())
|
||||
}
|
||||
@@ -734,7 +736,7 @@ func TestParseBucketPolicy(t *testing.T) {
|
||||
}
|
||||
// Test passes as expected, but the output values are verified for correctness here.
|
||||
if err == nil && testCase.shouldPass {
|
||||
if testCase.expectedPolicy.String() != actualAccessPolicy.String() {
|
||||
if !reflect.DeepEqual(testCase.expectedPolicy, actualAccessPolicy) {
|
||||
t.Errorf("Test %d: The expected statements from resource statement generator doesn't match the actual statements", i+1)
|
||||
}
|
||||
}
|
||||
@@ -751,8 +753,8 @@ func TestAWSRefererCondition(t *testing.T) {
|
||||
set.CreateStringSet("www.example.com",
|
||||
"http://www.example.com"))
|
||||
|
||||
requestConditionKeyMap := make(map[string]set.StringSet)
|
||||
requestConditionKeyMap["referer"] = set.CreateStringSet("www.example.com")
|
||||
requestConditionMap := make(policy.ConditionKeyMap)
|
||||
requestConditionMap["referer"] = set.CreateStringSet("www.example.com")
|
||||
|
||||
testCases := []struct {
|
||||
effect string
|
||||
@@ -782,20 +784,20 @@ func TestAWSRefererCondition(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions[test.conditionKey] = conditionsKeyMap
|
||||
|
||||
allowStatement := policyStatement{
|
||||
allowStatement := policy.Statement{
|
||||
Sid: "Testing AWS referer condition",
|
||||
Effect: test.effect,
|
||||
Principal: map[string]interface{}{
|
||||
"AWS": "*",
|
||||
Principal: policy.User{
|
||||
AWS: set.CreateStringSet("*"),
|
||||
},
|
||||
Resources: resource,
|
||||
Conditions: conditions,
|
||||
}
|
||||
|
||||
if result := bucketPolicyConditionMatch(requestConditionKeyMap, allowStatement); result != test.match {
|
||||
if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
|
||||
t.Errorf("Test %d - Expected conditons to evaluate to %v but got %v",
|
||||
i+1, test.match, result)
|
||||
}
|
||||
@@ -813,8 +815,8 @@ func TestAWSSourceIPCondition(t *testing.T) {
|
||||
set.CreateStringSet("54.240.143.0/24",
|
||||
"2001:DB8:1234:5678::/64"))
|
||||
|
||||
requestConditionKeyMap := make(map[string]set.StringSet)
|
||||
requestConditionKeyMap["ip"] = set.CreateStringSet("54.240.143.2")
|
||||
requestConditionMap := make(policy.ConditionKeyMap)
|
||||
requestConditionMap["ip"] = set.CreateStringSet("54.240.143.2")
|
||||
|
||||
testCases := []struct {
|
||||
effect string
|
||||
@@ -844,20 +846,20 @@ func TestAWSSourceIPCondition(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
conditions := make(map[string]map[string]set.StringSet)
|
||||
conditions := make(policy.ConditionMap)
|
||||
conditions[test.conditionKey] = conditionsKeyMap
|
||||
|
||||
allowStatement := policyStatement{
|
||||
allowStatement := policy.Statement{
|
||||
Sid: "Testing AWS referer condition",
|
||||
Effect: test.effect,
|
||||
Principal: map[string]interface{}{
|
||||
"AWS": "*",
|
||||
Principal: policy.User{
|
||||
AWS: set.CreateStringSet("*"),
|
||||
},
|
||||
Resources: resource,
|
||||
Conditions: conditions,
|
||||
}
|
||||
|
||||
if result := bucketPolicyConditionMatch(requestConditionKeyMap, allowStatement); result != test.match {
|
||||
if result := bucketPolicyConditionMatch(requestConditionMap, allowStatement); result != test.match {
|
||||
t.Errorf("Test %d - Expected conditons to evaluate to %v but got %v",
|
||||
i+1, test.match, result)
|
||||
}
|
||||
|
||||
+39
-27
@@ -20,7 +20,11 @@ import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -41,7 +45,7 @@ type bucketPolicies struct {
|
||||
rwMutex *sync.RWMutex
|
||||
|
||||
// Collection of 'bucket' policies.
|
||||
bucketPolicyConfigs map[string]*bucketPolicy
|
||||
bucketPolicyConfigs map[string]policy.BucketAccessPolicy
|
||||
}
|
||||
|
||||
// Represent a policy change
|
||||
@@ -51,11 +55,11 @@ type policyChange struct {
|
||||
IsRemove bool
|
||||
|
||||
// represents the new policy for the bucket
|
||||
BktPolicy *bucketPolicy
|
||||
BktPolicy policy.BucketAccessPolicy
|
||||
}
|
||||
|
||||
// Fetch bucket policy for a given bucket.
|
||||
func (bp bucketPolicies) GetBucketPolicy(bucket string) *bucketPolicy {
|
||||
func (bp bucketPolicies) GetBucketPolicy(bucket string) policy.BucketAccessPolicy {
|
||||
bp.rwMutex.RLock()
|
||||
defer bp.rwMutex.RUnlock()
|
||||
return bp.bucketPolicyConfigs[bucket]
|
||||
@@ -70,7 +74,7 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
|
||||
if pCh.IsRemove {
|
||||
delete(bp.bucketPolicyConfigs, bucket)
|
||||
} else {
|
||||
if pCh.BktPolicy == nil {
|
||||
if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
|
||||
return errInvalidArgument
|
||||
}
|
||||
bp.bucketPolicyConfigs[bucket] = pCh.BktPolicy
|
||||
@@ -79,19 +83,19 @@ func (bp *bucketPolicies) SetBucketPolicy(bucket string, pCh policyChange) error
|
||||
}
|
||||
|
||||
// Loads all bucket policies from persistent layer.
|
||||
func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolicy, err error) {
|
||||
func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.BucketAccessPolicy, err error) {
|
||||
// List buckets to proceed loading all notification configuration.
|
||||
buckets, err := objAPI.ListBuckets()
|
||||
errorIf(err, "Unable to list buckets.")
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list buckets.")
|
||||
return nil, errorCause(err)
|
||||
}
|
||||
|
||||
policies = make(map[string]*bucketPolicy)
|
||||
policies = make(map[string]policy.BucketAccessPolicy)
|
||||
var pErrs []error
|
||||
// Loads bucket policy.
|
||||
for _, bucket := range buckets {
|
||||
policy, pErr := readBucketPolicy(bucket.Name, objAPI)
|
||||
bp, pErr := readBucketPolicy(bucket.Name, objAPI)
|
||||
if pErr != nil {
|
||||
// net.Dial fails for rpc client or any
|
||||
// other unexpected errors during net.Dial.
|
||||
@@ -103,7 +107,7 @@ func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]*bucketPolic
|
||||
// Continue to load other bucket policies if possible.
|
||||
continue
|
||||
}
|
||||
policies[bucket.Name] = policy
|
||||
policies[bucket.Name] = bp
|
||||
}
|
||||
|
||||
// Look for any errors occurred while reading bucket policies.
|
||||
@@ -166,21 +170,20 @@ func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader
|
||||
|
||||
// readBucketPolicy - reads bucket policy for an input bucket, returns BucketPolicyNotFound
|
||||
// if bucket policy is not found. This function also parses the bucket policy into an object.
|
||||
func readBucketPolicy(bucket string, objAPI ObjectLayer) (*bucketPolicy, error) {
|
||||
func readBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPolicy, error) {
|
||||
// Read bucket policy JSON.
|
||||
bucketPolicyReader, err := readBucketPolicyJSON(bucket, objAPI)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return emptyBucketPolicy, err
|
||||
}
|
||||
|
||||
// Parse the saved policy.
|
||||
var policy = &bucketPolicy{}
|
||||
err = parseBucketPolicy(bucketPolicyReader, policy)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var bp policy.BucketAccessPolicy
|
||||
if err = parseBucketPolicy(bucketPolicyReader, &bp); err != nil {
|
||||
return emptyBucketPolicy, err
|
||||
|
||||
}
|
||||
return policy, nil
|
||||
return bp, nil
|
||||
}
|
||||
|
||||
// removeBucketPolicy - removes any previously written bucket policy. Returns BucketPolicyNotFound
|
||||
@@ -193,7 +196,8 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
if err := objAPI.DeleteObject(minioMetaBucket, policyPath); err != nil {
|
||||
err := objAPI.DeleteObject(minioMetaBucket, policyPath)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
|
||||
err = errorCause(err)
|
||||
if _, ok := err.(ObjectNotFound); ok {
|
||||
@@ -205,20 +209,27 @@ func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
|
||||
}
|
||||
|
||||
// writeBucketPolicy - save a bucket policy that is assumed to be validated.
|
||||
func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) error {
|
||||
func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAccessPolicy) error {
|
||||
buf, err := json.Marshal(bpy)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to marshal bucket policy '%v' to JSON", *bpy)
|
||||
errorIf(err, "Unable to marshal bucket policy '%#v' to JSON", bpy)
|
||||
return err
|
||||
}
|
||||
policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
|
||||
// Acquire a write lock on policy config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
if _, err := objAPI.PutObject(minioMetaBucket, policyPath, NewHashReader(bytes.NewReader(buf), int64(len(buf)), "", ""), nil); err != nil {
|
||||
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to set policy for the bucket %s", bucket)
|
||||
return errorCause(err)
|
||||
}
|
||||
|
||||
if _, err = objAPI.PutObject(minioMetaBucket, policyPath, hashReader, nil); err != nil {
|
||||
errorIf(err, "Unable to set policy for the bucket %s", bucket)
|
||||
return errorCause(err)
|
||||
}
|
||||
@@ -227,15 +238,15 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy *bucketPolicy) err
|
||||
|
||||
func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI ObjectLayer) APIErrorCode {
|
||||
// Parse bucket policy.
|
||||
var policy = &bucketPolicy{}
|
||||
err := parseBucketPolicy(bytes.NewReader(policyBytes), policy)
|
||||
var bktPolicy policy.BucketAccessPolicy
|
||||
err := parseBucketPolicy(bytes.NewReader(policyBytes), &bktPolicy)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to parse bucket policy.")
|
||||
return ErrInvalidPolicyDocument
|
||||
}
|
||||
|
||||
// Parse check bucket policy.
|
||||
if s3Error := checkBucketPolicyResources(bucket, policy); s3Error != ErrNone {
|
||||
if s3Error := checkBucketPolicyResources(bucket, bktPolicy); s3Error != ErrNone {
|
||||
return s3Error
|
||||
}
|
||||
|
||||
@@ -248,7 +259,7 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
|
||||
defer bucketLock.Unlock()
|
||||
|
||||
// Save bucket policy.
|
||||
if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, policy}, objAPI); err != nil {
|
||||
if err = persistAndNotifyBucketPolicyChange(bucket, policyChange{false, bktPolicy}, objAPI); err != nil {
|
||||
switch err.(type) {
|
||||
case BucketNameInvalid:
|
||||
return ErrInvalidBucketName
|
||||
@@ -267,11 +278,12 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
|
||||
// change. In-memory state is updated in response to the notification.
|
||||
func persistAndNotifyBucketPolicyChange(bucket string, pCh policyChange, objAPI ObjectLayer) error {
|
||||
if pCh.IsRemove {
|
||||
if err := removeBucketPolicy(bucket, objAPI); err != nil {
|
||||
err := removeBucketPolicy(bucket, objAPI)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
if pCh.BktPolicy == nil {
|
||||
if reflect.DeepEqual(pCh.BktPolicy, emptyBucketPolicy) {
|
||||
return errInvalidArgument
|
||||
}
|
||||
if err := writeBucketPolicy(bucket, objAPI, pCh.BktPolicy); err != nil {
|
||||
|
||||
+10
-2
@@ -24,6 +24,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
// Check for updates and print a notification message
|
||||
@@ -56,7 +57,7 @@ func initConfig() {
|
||||
// Config file does not exist, we create it fresh and return upon success.
|
||||
if isFile(getConfigFile()) {
|
||||
fatalIf(migrateConfig(), "Config migration failed.")
|
||||
fatalIf(loadConfig(), "Unable to load config version: '%s'.", v19)
|
||||
fatalIf(loadConfig(), "Unable to load config version: '%s'.", v20)
|
||||
} else {
|
||||
fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
|
||||
log.Println("Created minio configuration file successfully at " + getConfigDir())
|
||||
@@ -95,7 +96,7 @@ func handleCommonEnvVars() {
|
||||
accessKey := os.Getenv("MINIO_ACCESS_KEY")
|
||||
secretKey := os.Getenv("MINIO_SECRET_KEY")
|
||||
if accessKey != "" && secretKey != "" {
|
||||
cred, err := createCredential(accessKey, secretKey)
|
||||
cred, err := auth.CreateCredentials(accessKey, secretKey)
|
||||
fatalIf(err, "Invalid access/secret Key set in environment.")
|
||||
|
||||
// credential Envs are set globally.
|
||||
@@ -114,4 +115,11 @@ func handleCommonEnvVars() {
|
||||
globalIsEnvBrowser = true
|
||||
globalIsBrowserEnabled = bool(browserFlag)
|
||||
}
|
||||
|
||||
globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
|
||||
|
||||
globalDomainName = os.Getenv("MINIO_DOMAIN")
|
||||
if globalDomainName != "" {
|
||||
globalIsEnvDomainName = true
|
||||
}
|
||||
}
|
||||
|
||||
+116
-2
@@ -21,6 +21,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
)
|
||||
|
||||
@@ -147,7 +148,12 @@ func migrateConfig() error {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case v19:
|
||||
case "19":
|
||||
if err = migrateV19ToV20(); err != nil {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case "20":
|
||||
// No migration needed. this always points to current version.
|
||||
err = nil
|
||||
}
|
||||
@@ -190,7 +196,7 @@ func migrateV2ToV3() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
cred, err := createCredential(cv2.Credentials.AccessKey, cv2.Credentials.SecretKey)
|
||||
cred, err := auth.CreateCredentials(cv2.Credentials.AccessKey, cv2.Credentials.SecretKey)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Invalid credential in V2 configuration file. %v", err)
|
||||
}
|
||||
@@ -1478,3 +1484,111 @@ func migrateV18ToV19() error {
|
||||
log.Printf(configMigrateMSGTemplate, configFile, cv18.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV19ToV20() error {
|
||||
configFile := getConfigFile()
|
||||
|
||||
cv19 := &serverConfigV19{}
|
||||
_, err := quick.Load(configFile, cv19)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config version ‘18’. %v", err)
|
||||
}
|
||||
if cv19.Version != "19" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy over fields from V19 into V20 config struct
|
||||
srvConfig := &serverConfigV20{
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
}
|
||||
srvConfig.Version = "20"
|
||||
srvConfig.Credential = cv19.Credential
|
||||
srvConfig.Region = cv19.Region
|
||||
if srvConfig.Region == "" {
|
||||
// Region needs to be set for AWS Signature Version 4.
|
||||
srvConfig.Region = globalMinioDefaultRegion
|
||||
}
|
||||
|
||||
srvConfig.Logger.Console = cv19.Logger.Console
|
||||
srvConfig.Logger.File = cv19.Logger.File
|
||||
|
||||
// check and set notifiers config
|
||||
if len(cv19.Notify.AMQP) == 0 {
|
||||
srvConfig.Notify.AMQP = make(map[string]amqpNotify)
|
||||
srvConfig.Notify.AMQP["1"] = amqpNotify{}
|
||||
} else {
|
||||
// New deliveryMode parameter is added for AMQP,
|
||||
// default value is already 0, so nothing to
|
||||
// explicitly migrate here.
|
||||
srvConfig.Notify.AMQP = cv19.Notify.AMQP
|
||||
}
|
||||
if len(cv19.Notify.ElasticSearch) == 0 {
|
||||
srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
|
||||
srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.ElasticSearch = cv19.Notify.ElasticSearch
|
||||
}
|
||||
if len(cv19.Notify.Redis) == 0 {
|
||||
srvConfig.Notify.Redis = make(map[string]redisNotify)
|
||||
srvConfig.Notify.Redis["1"] = redisNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.Redis = cv19.Notify.Redis
|
||||
}
|
||||
if len(cv19.Notify.PostgreSQL) == 0 {
|
||||
srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
|
||||
srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.PostgreSQL = cv19.Notify.PostgreSQL
|
||||
}
|
||||
if len(cv19.Notify.Kafka) == 0 {
|
||||
srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
|
||||
srvConfig.Notify.Kafka["1"] = kafkaNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.Kafka = cv19.Notify.Kafka
|
||||
}
|
||||
if len(cv19.Notify.NATS) == 0 {
|
||||
srvConfig.Notify.NATS = make(map[string]natsNotify)
|
||||
srvConfig.Notify.NATS["1"] = natsNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.NATS = cv19.Notify.NATS
|
||||
}
|
||||
if len(cv19.Notify.Webhook) == 0 {
|
||||
srvConfig.Notify.Webhook = make(map[string]webhookNotify)
|
||||
srvConfig.Notify.Webhook["1"] = webhookNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.Webhook = cv19.Notify.Webhook
|
||||
}
|
||||
if len(cv19.Notify.MySQL) == 0 {
|
||||
srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
|
||||
srvConfig.Notify.MySQL["1"] = mySQLNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.MySQL = cv19.Notify.MySQL
|
||||
}
|
||||
if len(cv19.Notify.MQTT) == 0 {
|
||||
srvConfig.Notify.MQTT = make(map[string]mqttNotify)
|
||||
srvConfig.Notify.MQTT["1"] = mqttNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.MQTT = cv19.Notify.MQTT
|
||||
}
|
||||
|
||||
// Load browser config from existing config in the file.
|
||||
srvConfig.Browser = cv19.Browser
|
||||
|
||||
if err = quick.Save(configFile, srvConfig); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv19.Version, srvConfig.Version, err)
|
||||
}
|
||||
|
||||
log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -46,7 +46,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Check if config v1 is removed from filesystem
|
||||
if _, err := osStat(configPath); err == nil || !os.IsNotExist(err) {
|
||||
if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatal("Config V1 file is not purged")
|
||||
}
|
||||
|
||||
@@ -169,7 +169,7 @@ func TestServerConfigMigrateV2toV19(t *testing.T) {
|
||||
}
|
||||
|
||||
// Check the version number in the upgraded config file
|
||||
expectedVersion := v19
|
||||
expectedVersion := v20
|
||||
if serverConfig.Version != expectedVersion {
|
||||
t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
|
||||
}
|
||||
|
||||
+60
-38
@@ -16,7 +16,11 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "sync"
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
/////////////////// Config V1 ///////////////////
|
||||
type configV1 struct {
|
||||
@@ -92,8 +96,8 @@ type configV3 struct {
|
||||
Addr string `json:"address"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV3 `json:"logger"`
|
||||
@@ -122,8 +126,8 @@ type configV4 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV4 `json:"logger"`
|
||||
@@ -179,8 +183,8 @@ type configV5 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV5 `json:"logger"`
|
||||
@@ -209,8 +213,8 @@ type configV6 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
@@ -246,8 +250,8 @@ type serverConfigV7 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
@@ -265,8 +269,8 @@ type serverConfigV8 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
@@ -284,8 +288,8 @@ type serverConfigV9 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
@@ -310,8 +314,8 @@ type serverConfigV10 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV7 `json:"logger"`
|
||||
@@ -338,8 +342,8 @@ type serverConfigV11 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV7 `json:"logger"`
|
||||
@@ -354,8 +358,8 @@ type serverConfigV12 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV7 `json:"logger"`
|
||||
@@ -370,8 +374,8 @@ type serverConfigV13 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
@@ -386,9 +390,9 @@ type serverConfigV14 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
@@ -403,9 +407,9 @@ type serverConfigV15 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
@@ -420,9 +424,9 @@ type serverConfigV16 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
@@ -439,9 +443,9 @@ type serverConfigV17 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
@@ -458,9 +462,27 @@ type serverConfigV18 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV19 server configuration version '19' which is like
|
||||
// version '18' except it adds support for MQTT notifications.
|
||||
type serverConfigV19 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
@@ -22,29 +22,31 @@ import (
|
||||
"io/ioutil"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// Config version
|
||||
const v19 = "19"
|
||||
const v20 = "20"
|
||||
|
||||
var (
|
||||
// serverConfig server config.
|
||||
serverConfig *serverConfigV19
|
||||
serverConfig *serverConfigV20
|
||||
serverConfigMu sync.RWMutex
|
||||
)
|
||||
|
||||
// serverConfigV19 server configuration version '19' which is like
|
||||
// version '18' except it adds support for MQTT notifications.
|
||||
type serverConfigV19 struct {
|
||||
// serverConfigV20 server configuration version '20' which is like
|
||||
// version '19' except it adds support for VirtualHostDomain
|
||||
type serverConfigV20 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
@@ -54,7 +56,7 @@ type serverConfigV19 struct {
|
||||
}
|
||||
|
||||
// GetVersion get current config version.
|
||||
func (s *serverConfigV19) GetVersion() string {
|
||||
func (s *serverConfigV20) GetVersion() string {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
@@ -62,7 +64,7 @@ func (s *serverConfigV19) GetVersion() string {
|
||||
}
|
||||
|
||||
// SetRegion set a new region.
|
||||
func (s *serverConfigV19) SetRegion(region string) {
|
||||
func (s *serverConfigV20) SetRegion(region string) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
@@ -71,7 +73,7 @@ func (s *serverConfigV19) SetRegion(region string) {
|
||||
}
|
||||
|
||||
// GetRegion get current region.
|
||||
func (s *serverConfigV19) GetRegion() string {
|
||||
func (s *serverConfigV20) GetRegion() string {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
@@ -79,7 +81,7 @@ func (s *serverConfigV19) GetRegion() string {
|
||||
}
|
||||
|
||||
// SetCredentials set new credentials. SetCredential returns the previous credential.
|
||||
func (s *serverConfigV19) SetCredential(creds credential) (prevCred credential) {
|
||||
func (s *serverConfigV20) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
@@ -94,7 +96,7 @@ func (s *serverConfigV19) SetCredential(creds credential) (prevCred credential)
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s *serverConfigV19) GetCredential() credential {
|
||||
func (s *serverConfigV20) GetCredential() auth.Credentials {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
@@ -102,7 +104,7 @@ func (s *serverConfigV19) GetCredential() credential {
|
||||
}
|
||||
|
||||
// SetBrowser set if browser is enabled.
|
||||
func (s *serverConfigV19) SetBrowser(b bool) {
|
||||
func (s *serverConfigV20) SetBrowser(b bool) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
@@ -111,7 +113,7 @@ func (s *serverConfigV19) SetBrowser(b bool) {
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s *serverConfigV19) GetBrowser() bool {
|
||||
func (s *serverConfigV20) GetBrowser() bool {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
@@ -119,7 +121,7 @@ func (s *serverConfigV19) GetBrowser() bool {
|
||||
}
|
||||
|
||||
// Save config.
|
||||
func (s *serverConfigV19) Save() error {
|
||||
func (s *serverConfigV20) Save() error {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
@@ -127,10 +129,10 @@ func (s *serverConfigV19) Save() error {
|
||||
return quick.Save(getConfigFile(), s)
|
||||
}
|
||||
|
||||
func newServerConfigV19() *serverConfigV19 {
|
||||
srvCfg := &serverConfigV19{
|
||||
Version: v19,
|
||||
Credential: mustGetNewCredential(),
|
||||
func newServerConfigV20() *serverConfigV20 {
|
||||
srvCfg := &serverConfigV20{
|
||||
Version: v20,
|
||||
Credential: auth.MustGetNewCredentials(),
|
||||
Region: globalMinioDefaultRegion,
|
||||
Browser: true,
|
||||
Logger: &loggers{},
|
||||
@@ -167,7 +169,7 @@ func newServerConfigV19() *serverConfigV19 {
|
||||
// found, otherwise use default parameters
|
||||
func newConfig() error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfigV19()
|
||||
srvCfg := newServerConfigV20()
|
||||
|
||||
// If env is set override the credentials from config file.
|
||||
if globalIsEnvCreds {
|
||||
@@ -182,6 +184,10 @@ func newConfig() error {
|
||||
srvCfg.SetRegion(globalServerRegion)
|
||||
}
|
||||
|
||||
if globalIsEnvDomainName {
|
||||
srvCfg.Domain = globalDomainName
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
// Save the new config globally.
|
||||
// unlock the mutex.
|
||||
@@ -245,8 +251,8 @@ func checkDupJSONKeys(json string) error {
|
||||
}
|
||||
|
||||
// getValidConfig - returns valid server configuration
|
||||
func getValidConfig() (*serverConfigV19, error) {
|
||||
srvCfg := &serverConfigV19{
|
||||
func getValidConfig() (*serverConfigV20, error) {
|
||||
srvCfg := &serverConfigV20{
|
||||
Region: globalMinioDefaultRegion,
|
||||
Browser: true,
|
||||
}
|
||||
@@ -256,8 +262,8 @@ func getValidConfig() (*serverConfigV19, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if srvCfg.Version != v19 {
|
||||
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v19, srvCfg.Version)
|
||||
if srvCfg.Version != v20 {
|
||||
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v20, srvCfg.Version)
|
||||
}
|
||||
|
||||
// Load config file json and check for duplication json keys
|
||||
@@ -311,6 +317,10 @@ func loadConfig() error {
|
||||
srvCfg.SetRegion(globalServerRegion)
|
||||
}
|
||||
|
||||
if globalIsEnvDomainName {
|
||||
srvCfg.Domain = globalDomainName
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
serverConfigMu.Lock()
|
||||
serverConfig = srvCfg
|
||||
@@ -323,6 +333,9 @@ func loadConfig() error {
|
||||
if !globalIsEnvRegion {
|
||||
globalServerRegion = serverConfig.GetRegion()
|
||||
}
|
||||
if !globalIsEnvDomainName {
|
||||
globalDomainName = serverConfig.Domain
|
||||
}
|
||||
serverConfigMu.Unlock()
|
||||
|
||||
return nil
|
||||
@@ -117,8 +117,8 @@ func TestServerConfig(t *testing.T) {
|
||||
serverConfig.Logger.SetFile(fileLogger)
|
||||
|
||||
// Match version.
|
||||
if serverConfig.GetVersion() != v19 {
|
||||
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v19)
|
||||
if serverConfig.GetVersion() != v20 {
|
||||
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v20)
|
||||
}
|
||||
|
||||
// Attempt to save.
|
||||
@@ -149,6 +149,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
os.Setenv("MINIO_REGION", "us-west-1")
|
||||
defer os.Unsetenv("MINIO_REGION")
|
||||
|
||||
os.Setenv("MINIO_DOMAIN", "domain.com")
|
||||
defer os.Unsetenv("MINIO_DOMAIN")
|
||||
|
||||
defer resetGlobalIsEnvs()
|
||||
|
||||
// Get test root.
|
||||
@@ -189,6 +192,9 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
|
||||
}
|
||||
|
||||
if serverConfig.Domain != "domain.com" {
|
||||
t.Errorf("Expecting Domain to be `domain.com` found " + serverConfig.Domain)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCheckDupJSONKeys(t *testing.T) {
|
||||
@@ -231,7 +237,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
|
||||
configPath := filepath.Join(rootPath, minioConfigFile)
|
||||
|
||||
v := v19
|
||||
v := v20
|
||||
|
||||
testCases := []struct {
|
||||
configData string
|
||||
@@ -1,100 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestMustGetNewCredential(t *testing.T) {
|
||||
cred := mustGetNewCredential()
|
||||
if !cred.IsValid() {
|
||||
t.Fatalf("Failed to get new valid credential")
|
||||
}
|
||||
if len(cred.SecretKey) != secretKeyMaxLen {
|
||||
t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLen)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCredential(t *testing.T) {
|
||||
cred := mustGetNewCredential()
|
||||
testCases := []struct {
|
||||
accessKey string
|
||||
secretKey string
|
||||
expectedResult bool
|
||||
expectedErr error
|
||||
}{
|
||||
// Access key too small (min 5 chars).
|
||||
{"user", "pass", false, errInvalidAccessKeyLength},
|
||||
// Long access key is ok.
|
||||
{"user123456789012345678901234567890", "password", true, nil},
|
||||
// Secret key too small (min 8 chars).
|
||||
{"myuser", "pass", false, errInvalidSecretKeyLength},
|
||||
// Long secret key is ok.
|
||||
{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", true, nil},
|
||||
{"myuser", "mypassword", true, nil},
|
||||
{cred.AccessKey, cred.SecretKey, true, nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
cred, err := createCredential(testCase.accessKey, testCase.secretKey)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
|
||||
if testCase.expectedResult != cred.IsValid() {
|
||||
t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, cred.IsValid())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCredentialEqual(t *testing.T) {
|
||||
cred := mustGetNewCredential()
|
||||
testCases := []struct {
|
||||
cred credential
|
||||
ccred credential
|
||||
expectedResult bool
|
||||
}{
|
||||
// Empty compare credential
|
||||
{cred, credential{}, false},
|
||||
// Empty credential
|
||||
{credential{}, cred, false},
|
||||
// Two different credentials
|
||||
{cred, mustGetNewCredential(), false},
|
||||
// Access key is different in compare credential.
|
||||
{cred, credential{AccessKey: "myuser", SecretKey: cred.SecretKey}, false},
|
||||
// Secret key is different in compare credential.
|
||||
{cred, credential{AccessKey: cred.AccessKey, SecretKey: "mypassword"}, false},
|
||||
// secretHashKey is missing in compare credential.
|
||||
{cred, credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, true},
|
||||
// secretHashKey is missing in credential.
|
||||
{credential{AccessKey: cred.AccessKey, SecretKey: cred.SecretKey}, cred, true},
|
||||
// Same credentials.
|
||||
{cred, cred, true},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
result := testCase.cred.Equal(testCase.ccred)
|
||||
if result != testCase.expectedResult {
|
||||
t.Fatalf("cred: expected: %v, got: %v", testCase.expectedResult, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,339 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/md5"
|
||||
"crypto/rand"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
var (
|
||||
// AWS errors for invalid SSE-C requests.
|
||||
errInsecureSSERequest = errors.New("Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection")
|
||||
errEncryptedObject = errors.New("The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object")
|
||||
errInvalidSSEAlgorithm = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm")
|
||||
errMissingSSEKey = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key")
|
||||
errInvalidSSEKey = errors.New("The secret key was invalid for the specified algorithm")
|
||||
errMissingSSEKeyMD5 = errors.New("Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key")
|
||||
errSSEKeyMD5Mismatch = errors.New("The calculated MD5 hash of the key did not match the hash that was provided")
|
||||
errSSEKeyMismatch = errors.New("The client provided key does not match the key provided when the object was encrypted") // this msg is not shown to the client
|
||||
|
||||
// Additional Minio errors for SSE-C requests.
|
||||
errObjectTampered = errors.New("The requested object was modified and may be compromised")
|
||||
)
|
||||
|
||||
const (
|
||||
// SSECustomerAlgorithm is the AWS SSE-C algorithm HTTP header key.
|
||||
SSECustomerAlgorithm = "X-Amz-Server-Side-Encryption-Customer-Algorithm"
|
||||
// SSECustomerKey is the AWS SSE-C encryption key HTTP header key.
|
||||
SSECustomerKey = "X-Amz-Server-Side-Encryption-Customer-Key"
|
||||
// SSECustomerKeyMD5 is the AWS SSE-C encryption key MD5 HTTP header key.
|
||||
SSECustomerKeyMD5 = "X-Amz-Server-Side-Encryption-Customer-Key-MD5"
|
||||
)
|
||||
|
||||
const (
|
||||
// SSECustomerKeySize is the size of valid client provided encryption keys in bytes.
|
||||
// Currently AWS supports only AES256. So the SSE-C key size is fixed to 32 bytes.
|
||||
SSECustomerKeySize = 32
|
||||
|
||||
// SSECustomerAlgorithmAES256 the only valid S3 SSE-C encryption algorithm identifier.
|
||||
SSECustomerAlgorithmAES256 = "AES256"
|
||||
)
|
||||
|
||||
// SSE-C key derivation, key verification and key update:
|
||||
// H: Hash function [32 = |H(m)|]
|
||||
// AE: authenticated encryption scheme, AD: authenticated decryption scheme [m = AD(k, AE(k, m))]
|
||||
//
|
||||
// Key derivation:
|
||||
// Input:
|
||||
// key := 32 bytes # client provided key
|
||||
// Re, Rm := 32 bytes, 32 bytes # uniformly random
|
||||
//
|
||||
// Seal:
|
||||
// k := H(key || Re) # object encryption key
|
||||
// r := H(Rm) # save as object metadata [ServerSideEncryptionIV]
|
||||
// KeK := H(key || r) # key encryption key
|
||||
// K := AE(KeK, k) # save as object metadata [ServerSideEncryptionSealedKey]
|
||||
// ------------------------------------------------------------------------------------------------
|
||||
// Key verification:
|
||||
// Input:
|
||||
// key := 32 bytes # client provided key
|
||||
// r := 32 bytes # object metadata [ServerSideEncryptionIV]
|
||||
// K := 32 bytes # object metadata [ServerSideEncryptionSealedKey]
|
||||
//
|
||||
// Open:
|
||||
// KeK := H(key || r) # key encryption key
|
||||
// k := AD(Kek, K) # object encryption key
|
||||
// -------------------------------------------------------------------------------------------------
|
||||
// Key update:
|
||||
// Input:
|
||||
// key := 32 bytes # old client provided key
|
||||
// key' := 32 bytes # new client provided key
|
||||
// Rm := 32 bytes # uniformly random
|
||||
// r := 32 bytes # object metadata [ServerSideEncryptionIV]
|
||||
// K := 32 bytes # object metadata [ServerSideEncryptionSealedKey]
|
||||
//
|
||||
// Update:
|
||||
// 1. open:
|
||||
// KeK := H(key || r) # key encryption key
|
||||
// k := AD(Kek, K) # object encryption key
|
||||
// 2. seal:
|
||||
// r' := H(Rm) # save as object metadata [ServerSideEncryptionIV]
|
||||
// KeK' := H(key' || r') # new key encryption key
|
||||
// K' := AE(KeK', k) # save as object metadata [ServerSideEncryptionSealedKey]
|
||||
|
||||
const (
|
||||
// ServerSideEncryptionIV is a 32 byte randomly generated IV used to derive an
|
||||
// unique key encryption key from the client provided key. The combination of this value
|
||||
// and the client-provided key MUST be unique.
|
||||
ServerSideEncryptionIV = ReservedMetadataPrefix + "Server-Side-Encryption-Iv"
|
||||
|
||||
// ServerSideEncryptionSealAlgorithm identifies a combination of a cryptographic hash function and
|
||||
// an authenticated en/decryption scheme to seal the object encryption key.
|
||||
ServerSideEncryptionSealAlgorithm = ReservedMetadataPrefix + "Server-Side-Encryption-Seal-Algorithm"
|
||||
|
||||
// ServerSideEncryptionSealedKey is the sealed object encryption key. The sealed key can be decrypted
|
||||
// by the key encryption key derived from the client provided key and the server-side-encryption IV.
|
||||
ServerSideEncryptionSealedKey = ReservedMetadataPrefix + "Server-Side-Encryption-Sealed-Key"
|
||||
)
|
||||
|
||||
// SSESealAlgorithmDareSha256 specifies DARE as authenticated en/decryption scheme and SHA256 as cryptographic
|
||||
// hash function.
|
||||
const SSESealAlgorithmDareSha256 = "DARE-SHA256"
|
||||
|
||||
// IsSSECustomerRequest returns true if the given HTTP header
|
||||
// contains server-side-encryption with customer provided key fields.
|
||||
func IsSSECustomerRequest(header http.Header) bool {
|
||||
return header.Get(SSECustomerAlgorithm) != "" || header.Get(SSECustomerKey) != "" || header.Get(SSECustomerKeyMD5) != ""
|
||||
}
|
||||
|
||||
// ParseSSECustomerRequest parses the SSE-C header fields of the provided request.
|
||||
// It returns the client provided key on success.
|
||||
func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
|
||||
if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
|
||||
// we cannot use r.TLS == nil here because Go's http implementation reflects on
|
||||
// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
|
||||
// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
|
||||
// will always fail -> r.TLS is always nil even for TLS requests.
|
||||
return nil, errInsecureSSERequest
|
||||
}
|
||||
header := r.Header
|
||||
if algorithm := header.Get(SSECustomerAlgorithm); algorithm != SSECustomerAlgorithmAES256 {
|
||||
return nil, errInvalidSSEAlgorithm
|
||||
}
|
||||
if header.Get(SSECustomerKey) == "" {
|
||||
return nil, errMissingSSEKey
|
||||
}
|
||||
if header.Get(SSECustomerKeyMD5) == "" {
|
||||
return nil, errMissingSSEKeyMD5
|
||||
}
|
||||
|
||||
key, err = base64.StdEncoding.DecodeString(header.Get(SSECustomerKey))
|
||||
if err != nil {
|
||||
return nil, errInvalidSSEKey
|
||||
}
|
||||
header.Del(SSECustomerKey) // make sure we do not save the key by accident
|
||||
|
||||
if len(key) != SSECustomerKeySize {
|
||||
return nil, errInvalidSSEKey
|
||||
}
|
||||
|
||||
keyMD5, err := base64.StdEncoding.DecodeString(header.Get(SSECustomerKeyMD5))
|
||||
if err != nil {
|
||||
return nil, errSSEKeyMD5Mismatch
|
||||
}
|
||||
if md5Sum := md5.Sum(key); !bytes.Equal(md5Sum[:], keyMD5) {
|
||||
return nil, errSSEKeyMD5Mismatch
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// EncryptRequest takes the client provided content and encrypts the data
|
||||
// with the client provided key. It also marks the object as client-side-encrypted
|
||||
// and sets the correct headers.
|
||||
func EncryptRequest(content io.Reader, r *http.Request, metadata map[string]string) (io.Reader, error) {
|
||||
key, err := ParseSSECustomerRequest(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
|
||||
|
||||
// security notice:
|
||||
// - If the first 32 bytes of the random value are ever repeated under the same client-provided
|
||||
// key the encrypted object will not be tamper-proof. [ P(coll) ~= 1 / 2^(256 / 2)]
|
||||
// - If the last 32 bytes of the random value are ever repeated under the same client-provided
|
||||
// key an adversary may be able to extract the object encryption key. This depends on the
|
||||
// authenticated en/decryption scheme. The DARE format will generate an 8 byte nonce which must
|
||||
// be repeated in addition to reveal the object encryption key.
|
||||
// [ P(coll) ~= 1 / 2^((256 + 64) / 2) ]
|
||||
nonce := make([]byte, 64) // generate random values for key derivation
|
||||
if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sha := sha256.New() // derive object encryption key
|
||||
sha.Write(key)
|
||||
sha.Write(nonce[:32])
|
||||
objectEncryptionKey := sha.Sum(nil)
|
||||
|
||||
iv := sha256.Sum256(nonce[32:]) // derive key encryption key
|
||||
sha = sha256.New()
|
||||
sha.Write(key)
|
||||
sha.Write(iv[:])
|
||||
keyEncryptionKey := sha.Sum(nil)
|
||||
|
||||
sealedKey := bytes.NewBuffer(nil) // sealedKey := 16 byte header + 32 byte payload + 16 byte tag
|
||||
n, err := sio.Encrypt(sealedKey, bytes.NewReader(objectEncryptionKey), sio.Config{
|
||||
Key: keyEncryptionKey,
|
||||
})
|
||||
if n != 64 || err != nil {
|
||||
return nil, errors.New("failed to seal object encryption key") // if this happens there's a bug in the code (may panic ?)
|
||||
}
|
||||
|
||||
reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey})
|
||||
if err != nil {
|
||||
return nil, errInvalidSSEKey
|
||||
}
|
||||
|
||||
metadata[ServerSideEncryptionIV] = base64.StdEncoding.EncodeToString(iv[:])
|
||||
metadata[ServerSideEncryptionSealAlgorithm] = SSESealAlgorithmDareSha256
|
||||
metadata[ServerSideEncryptionSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Bytes())
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
// DecryptRequest decrypts the object with the client provided key. It also removes
|
||||
// the client-side-encryption metadata from the object and sets the correct headers.
|
||||
func DecryptRequest(client io.Writer, r *http.Request, metadata map[string]string) (io.WriteCloser, error) {
|
||||
key, err := ParseSSECustomerRequest(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
delete(metadata, SSECustomerKey) // make sure we do not save the key by accident
|
||||
|
||||
if metadata[ServerSideEncryptionSealAlgorithm] != SSESealAlgorithmDareSha256 { // currently DARE-SHA256 is the only option
|
||||
return nil, errObjectTampered
|
||||
}
|
||||
iv, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionIV])
|
||||
if err != nil || len(iv) != 32 {
|
||||
return nil, errObjectTampered
|
||||
}
|
||||
sealedKey, err := base64.StdEncoding.DecodeString(metadata[ServerSideEncryptionSealedKey])
|
||||
if err != nil || len(sealedKey) != 64 {
|
||||
return nil, errObjectTampered
|
||||
}
|
||||
|
||||
sha := sha256.New() // derive key encryption key
|
||||
sha.Write(key)
|
||||
sha.Write(iv)
|
||||
keyEncryptionKey := sha.Sum(nil)
|
||||
|
||||
objectEncryptionKey := bytes.NewBuffer(nil) // decrypt object encryption key
|
||||
n, err := sio.Decrypt(objectEncryptionKey, bytes.NewReader(sealedKey), sio.Config{
|
||||
Key: keyEncryptionKey,
|
||||
})
|
||||
if n != 32 || err != nil {
|
||||
// Either the provided key does not match or the object was tampered.
|
||||
// To provide strict AWS S3 compatibility we return: access denied.
|
||||
return nil, errSSEKeyMismatch
|
||||
}
|
||||
|
||||
writer, err := sio.DecryptWriter(client, sio.Config{Key: objectEncryptionKey.Bytes()})
|
||||
if err != nil {
|
||||
return nil, errInvalidSSEKey
|
||||
}
|
||||
|
||||
delete(metadata, ServerSideEncryptionIV)
|
||||
delete(metadata, ServerSideEncryptionSealAlgorithm)
|
||||
delete(metadata, ServerSideEncryptionSealedKey)
|
||||
return writer, nil
|
||||
}
|
||||
|
||||
// IsEncrypted returns true if the object is marked as encrypted.
|
||||
func (o *ObjectInfo) IsEncrypted() bool {
|
||||
if _, ok := o.UserDefined[ServerSideEncryptionIV]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := o.UserDefined[ServerSideEncryptionSealAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := o.UserDefined[ServerSideEncryptionSealedKey]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// DecryptedSize returns the size of the object after decryption in bytes.
|
||||
// It returns an error if the object is not encrypted or marked as encrypted
|
||||
// but has an invalid size.
|
||||
// DecryptedSize panics if the referred object is not encrypted.
|
||||
func (o *ObjectInfo) DecryptedSize() (int64, error) {
|
||||
if !o.IsEncrypted() {
|
||||
panic("cannot compute decrypted size of an object which is not encrypted")
|
||||
}
|
||||
if o.Size == 0 {
|
||||
return o.Size, nil
|
||||
}
|
||||
size := (o.Size / (32 + 64*1024)) * (64 * 1024)
|
||||
if mod := o.Size % (32 + 64*1024); mod > 0 {
|
||||
if mod < 33 {
|
||||
return -1, errObjectTampered // object is not 0 size but smaller than the smallest valid encrypted object
|
||||
}
|
||||
size += mod - 32
|
||||
}
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// EncryptedSize returns the size of the object after encryption.
|
||||
// An encrypted object is always larger than a plain object
|
||||
// except for zero size objects.
|
||||
func (o *ObjectInfo) EncryptedSize() int64 {
|
||||
size := (o.Size / (64 * 1024)) * (32 + 64*1024)
|
||||
if mod := o.Size % (64 * 1024); mod > 0 {
|
||||
size += mod + 32
|
||||
}
|
||||
return size
|
||||
}
|
||||
|
||||
// DecryptObjectInfo tries to decrypt the provided object if it is encrypted.
|
||||
// It fails if the object is encrypted and the HTTP headers don't contain
|
||||
// SSE-C headers or the object is not encrypted but SSE-C headers are provided. (AWS behavior)
|
||||
// DecryptObjectInfo returns 'ErrNone' if the object is not encrypted or the
|
||||
// decryption succeeded.
|
||||
//
|
||||
// DecryptObjectInfo also returns whether the object is encrypted or not.
|
||||
func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErrorCode, encrypted bool) {
|
||||
if apiErr, encrypted = ErrNone, info.IsEncrypted(); !encrypted && IsSSECustomerRequest(headers) {
|
||||
apiErr = ErrInvalidEncryptionParameters
|
||||
} else if encrypted {
|
||||
if !IsSSECustomerRequest(headers) {
|
||||
apiErr = ErrSSEEncryptedObject
|
||||
return
|
||||
}
|
||||
var err error
|
||||
if info.Size, err = info.DecryptedSize(); err != nil {
|
||||
apiErr = toAPIErrorCode(err)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,390 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var isSSECustomerRequestTests = []struct {
|
||||
headers map[string]string
|
||||
sseRequest bool
|
||||
}{
|
||||
{headers: map[string]string{SSECustomerAlgorithm: "AES256", SSECustomerKey: "key", SSECustomerKeyMD5: "md5"}, sseRequest: true}, // 0
|
||||
{headers: map[string]string{SSECustomerAlgorithm: "AES256"}, sseRequest: true}, // 1
|
||||
{headers: map[string]string{SSECustomerKey: "key"}, sseRequest: true}, // 2
|
||||
{headers: map[string]string{SSECustomerKeyMD5: "md5"}, sseRequest: true}, // 3
|
||||
{headers: map[string]string{}, sseRequest: false}, // 4
|
||||
{headers: map[string]string{SSECustomerAlgorithm + " ": "AES256", " " + SSECustomerKey: "key", SSECustomerKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
|
||||
{headers: map[string]string{SSECustomerAlgorithm: "", SSECustomerKey: "", SSECustomerKeyMD5: ""}, sseRequest: false}, // 6
|
||||
}
|
||||
|
||||
func TestIsSSECustomerRequest(t *testing.T) {
|
||||
for i, test := range isSSECustomerRequestTests {
|
||||
headers := http.Header{}
|
||||
for k, v := range test.headers {
|
||||
headers.Set(k, v)
|
||||
}
|
||||
if IsSSECustomerRequest(headers) != test.sseRequest {
|
||||
t.Errorf("Test %d: Expected IsSSECustomerRequest to return %v", i, test.sseRequest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var parseSSECustomerRequestTests = []struct {
|
||||
headers map[string]string
|
||||
useTLS bool
|
||||
err error
|
||||
}{
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
useTLS: true, err: nil,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
useTLS: false, err: errInsecureSSERequest,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES 256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
useTLS: true, err: errInvalidSSEAlgorithm,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
|
||||
SSECustomerKeyMD5: "H+jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
useTLS: true, err: errSSEKeyMD5Mismatch,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
|
||||
SSECustomerKeyMD5: "H+jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
useTLS: true, err: errInvalidSSEKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
|
||||
SSECustomerKeyMD5: " +jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
useTLS: true, err: errSSEKeyMD5Mismatch,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
|
||||
SSECustomerKeyMD5: "9KPgDdZNTHimuYCwnJTp5g==",
|
||||
},
|
||||
useTLS: true, err: errInvalidSSEKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "", // 7
|
||||
SSECustomerKeyMD5: "9KPgDdZNTHimuYCwnJTp5g==",
|
||||
},
|
||||
useTLS: true, err: errMissingSSEKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
|
||||
SSECustomerKeyMD5: "",
|
||||
},
|
||||
useTLS: true, err: errMissingSSEKeyMD5,
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseSSECustomerRequest(t *testing.T) {
|
||||
defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
|
||||
for i, test := range parseSSECustomerRequestTests {
|
||||
headers := http.Header{}
|
||||
for k, v := range test.headers {
|
||||
headers.Set(k, v)
|
||||
}
|
||||
request := &http.Request{}
|
||||
request.Header = headers
|
||||
globalIsSSL = test.useTLS
|
||||
|
||||
_, err := ParseSSECustomerRequest(request)
|
||||
if err != test.err {
|
||||
t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
|
||||
}
|
||||
key := request.Header.Get(SSECustomerKey)
|
||||
if (err == nil || err == errSSEKeyMD5Mismatch) && key != "" {
|
||||
t.Errorf("Test %d: Client key survived parsing - found key: %v", i, key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var encryptedSizeTests = []struct {
|
||||
size, encsize int64
|
||||
}{
|
||||
{size: 0, encsize: 0}, // 0
|
||||
{size: 1, encsize: 33}, // 1
|
||||
{size: 1024, encsize: 1024 + 32}, // 2
|
||||
{size: 2 * 64 * 1024, encsize: 2 * (64*1024 + 32)}, // 3
|
||||
{size: 100*64*1024 + 1, encsize: 100*(64*1024+32) + 33}, // 4
|
||||
{size: 64*1024 + 1, encsize: (64*1024 + 32) + 33}, // 5
|
||||
{size: 5 * 1024 * 1024 * 1024, encsize: 81920 * (64*1024 + 32)}, // 6
|
||||
}
|
||||
|
||||
func TestEncryptedSize(t *testing.T) {
|
||||
for i, test := range encryptedSizeTests {
|
||||
objInfo := ObjectInfo{Size: test.size}
|
||||
if size := objInfo.EncryptedSize(); test.encsize != size {
|
||||
t.Errorf("Test %d: got encrypted size: #%d want: #%d", i, size, test.encsize)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var decryptSSECustomerObjectInfoTests = []struct {
|
||||
encsize, size int64
|
||||
err error
|
||||
}{
|
||||
{encsize: 0, size: 0, err: nil}, // 0
|
||||
{encsize: 33, size: 1, err: nil}, // 1
|
||||
{encsize: 1024 + 32, size: 1024, err: nil}, // 2
|
||||
{encsize: 2 * (64*1024 + 32), size: 2 * 64 * 1024, err: nil}, // 3
|
||||
{encsize: 100*(64*1024+32) + 33, size: 100*64*1024 + 1, err: nil}, // 4
|
||||
{encsize: (64*1024 + 32) + 33, size: 64*1024 + 1, err: nil}, // 5
|
||||
{encsize: 81920 * (64*1024 + 32), size: 5 * 1024 * 1024 * 1024, err: nil}, // 6
|
||||
{encsize: 0, size: 0, err: nil}, // 7
|
||||
{encsize: 64*1024 + 32 + 31, size: 0, err: errObjectTampered}, // 8
|
||||
}
|
||||
|
||||
func TestDecryptedSize(t *testing.T) {
|
||||
for i, test := range decryptSSECustomerObjectInfoTests {
|
||||
objInfo := ObjectInfo{Size: test.encsize}
|
||||
objInfo.UserDefined = map[string]string{
|
||||
ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
|
||||
}
|
||||
|
||||
size, err := objInfo.DecryptedSize()
|
||||
if err != test.err || (size != test.size && err == nil) {
|
||||
t.Errorf("Test %d: decryption returned: %v want: %v", i, err, test.err)
|
||||
}
|
||||
if err == nil && size != test.size {
|
||||
t.Errorf("Test %d: got decrypted size: #%d want: #%d", i, size, test.size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var encryptRequestTests = []struct {
|
||||
header map[string]string
|
||||
metadata map[string]string
|
||||
}{
|
||||
{
|
||||
header: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
},
|
||||
{
|
||||
header: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestEncryptRequest(t *testing.T) {
|
||||
defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
|
||||
globalIsSSL = true
|
||||
for i, test := range encryptRequestTests {
|
||||
content := bytes.NewReader(make([]byte, 64))
|
||||
req := &http.Request{Header: http.Header{}}
|
||||
for k, v := range test.header {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
_, err := EncryptRequest(content, req, test.metadata)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
|
||||
}
|
||||
if key, ok := test.metadata[SSECustomerKey]; ok {
|
||||
t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
|
||||
}
|
||||
if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; !ok {
|
||||
t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
|
||||
}
|
||||
if iv, ok := test.metadata[ServerSideEncryptionIV]; !ok {
|
||||
t.Errorf("Test %d: ServerSideEncryptionIV must be part of metadata: %v", i, iv)
|
||||
}
|
||||
if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; !ok {
|
||||
t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var decryptRequestTests = []struct {
|
||||
header map[string]string
|
||||
metadata map[string]string
|
||||
shouldFail bool
|
||||
}{
|
||||
{
|
||||
header: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=",
|
||||
SSECustomerKeyMD5: "7PpPLAK26ONlVUGOWlusfg==",
|
||||
},
|
||||
metadata: map[string]string{
|
||||
ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
|
||||
ServerSideEncryptionIV: "7nQqotA8xgrPx6QK7Ap3GCfjKitqJSrGP7xzgErSJlw=",
|
||||
ServerSideEncryptionSealedKey: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA==",
|
||||
},
|
||||
shouldFail: false,
|
||||
},
|
||||
{
|
||||
header: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{
|
||||
ServerSideEncryptionSealAlgorithm: "HMAC-SHA3",
|
||||
ServerSideEncryptionIV: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
ServerSideEncryptionSealedKey: "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
|
||||
},
|
||||
shouldFail: true,
|
||||
},
|
||||
{
|
||||
header: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{
|
||||
ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
|
||||
ServerSideEncryptionIV: "RrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
ServerSideEncryptionSealedKey: "SY5E9AvI2tI7/nUrUAssIGE32Hcs4rR9z/CUuPqu5N4=",
|
||||
},
|
||||
shouldFail: true,
|
||||
},
|
||||
{
|
||||
header: map[string]string{
|
||||
SSECustomerAlgorithm: "AES256",
|
||||
SSECustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
SSECustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{
|
||||
ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256,
|
||||
ServerSideEncryptionIV: "XAm0dRrJsEsyPb1UuFNezv1bl9ehxuYsgUVC/MUctE2k=",
|
||||
ServerSideEncryptionSealedKey: "SY5E9AvI2tI7/nUrUAssIGE32Hds4rR9z/CUuPqu5N4=",
|
||||
},
|
||||
shouldFail: true,
|
||||
},
|
||||
}
|
||||
|
||||
func TestDecryptRequest(t *testing.T) {
|
||||
defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
|
||||
globalIsSSL = true
|
||||
for i, test := range decryptRequestTests {
|
||||
client := bytes.NewBuffer(nil)
|
||||
req := &http.Request{Header: http.Header{}}
|
||||
for k, v := range test.header {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
_, err := DecryptRequest(client, req, test.metadata)
|
||||
if err != nil && !test.shouldFail {
|
||||
t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
|
||||
}
|
||||
if key, ok := test.metadata[SSECustomerKey]; ok {
|
||||
t.Errorf("Test %d: Client provided key survived in metadata - key: %s", i, key)
|
||||
}
|
||||
if kdf, ok := test.metadata[ServerSideEncryptionSealAlgorithm]; ok && !test.shouldFail {
|
||||
t.Errorf("Test %d: ServerSideEncryptionKDF should not be part of metadata: %v", i, kdf)
|
||||
}
|
||||
if iv, ok := test.metadata[ServerSideEncryptionIV]; ok && !test.shouldFail {
|
||||
t.Errorf("Test %d: ServerSideEncryptionIV should not be part of metadata: %v", i, iv)
|
||||
}
|
||||
if mac, ok := test.metadata[ServerSideEncryptionSealedKey]; ok && !test.shouldFail {
|
||||
t.Errorf("Test %d: ServerSideEncryptionKeyMAC should not be part of metadata: %v", i, mac)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var decryptObjectInfoTests = []struct {
|
||||
info ObjectInfo
|
||||
headers http.Header
|
||||
expErr APIErrorCode
|
||||
}{
|
||||
{
|
||||
info: ObjectInfo{Size: 100},
|
||||
headers: http.Header{},
|
||||
expErr: ErrNone,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
|
||||
headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
|
||||
expErr: ErrNone,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 0, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
|
||||
headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
|
||||
expErr: ErrNone,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
|
||||
headers: http.Header{},
|
||||
expErr: ErrSSEEncryptedObject,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{}},
|
||||
headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
|
||||
expErr: ErrInvalidEncryptionParameters,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 31, UserDefined: map[string]string{ServerSideEncryptionSealAlgorithm: SSESealAlgorithmDareSha256}},
|
||||
headers: http.Header{SSECustomerAlgorithm: []string{SSECustomerAlgorithmAES256}},
|
||||
expErr: ErrObjectTampered,
|
||||
},
|
||||
}
|
||||
|
||||
func TestDecryptObjectInfo(t *testing.T) {
|
||||
for i, test := range decryptObjectInfoTests {
|
||||
if err, encrypted := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
|
||||
t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
|
||||
} else if enc := test.info.IsEncrypted(); encrypted && enc != encrypted {
|
||||
t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
|
||||
} else if !encrypted && enc != encrypted {
|
||||
t.Errorf("Test %d: Decryption thinks object is not encrypted but it is", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"net/url"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -130,6 +131,26 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
return ep, fmt.Errorf("empty or root path is not supported in URL endpoint")
|
||||
}
|
||||
|
||||
// On windows having a preceding "/" will cause problems, if the
|
||||
// command line already has C:/<export-folder/ in it. Final resulting
|
||||
// path on windows might become C:/C:/ this will cause problems
|
||||
// of starting minio server properly in distributed mode on windows.
|
||||
// As a special case make sure to trim the separator.
|
||||
|
||||
// NOTE: It is also perfectly fine for windows users to have a path
|
||||
// without C:/ since at that point we treat it as relative path
|
||||
// and obtain the full filesystem path as well. Providing C:/
|
||||
// style is necessary to provide paths other than C:/,
|
||||
// such as F:/, D:/ etc.
|
||||
//
|
||||
// Another additional benefit here is that this style also
|
||||
// supports providing \\host\share support as well.
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
if filepath.VolumeName(u.Path[1:]) != "" {
|
||||
u.Path = u.Path[1:]
|
||||
}
|
||||
}
|
||||
|
||||
isLocal, err = isLocalHost(host)
|
||||
if err != nil {
|
||||
return ep, err
|
||||
@@ -419,6 +440,29 @@ func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, S
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// GetLocalPeer - returns local peer value, returns globalMinioAddr
|
||||
// for FS and Erasure mode. In case of distributed server return
|
||||
// the first element from the set of peers which indicate that
|
||||
// they are local. There is always one entry that is local
|
||||
// even with repeated server endpoints.
|
||||
func GetLocalPeer(endpoints EndpointList) (localPeer string) {
|
||||
peerSet := set.NewStringSet()
|
||||
for _, endpoint := range endpoints {
|
||||
if endpoint.Type() != URLEndpointType {
|
||||
continue
|
||||
}
|
||||
if endpoint.IsLocal && endpoint.Host != "" {
|
||||
peerSet.Add(endpoint.Host)
|
||||
}
|
||||
}
|
||||
if peerSet.IsEmpty() {
|
||||
// If local peer is empty can happen in FS or Erasure coded mode.
|
||||
// then set the value to globalMinioAddr instead.
|
||||
return globalMinioAddr
|
||||
}
|
||||
return peerSet.ToSlice()[0]
|
||||
}
|
||||
|
||||
// GetRemotePeers - get hosts information other than this minio service.
|
||||
func GetRemotePeers(endpoints EndpointList) []string {
|
||||
peerSet := set.NewStringSet()
|
||||
|
||||
@@ -330,6 +330,38 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests get local peer functionality, local peer is supposed to only return one entry per minio service.
|
||||
// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001
|
||||
// is considered a remote service from localhost:9000 perspective.
|
||||
func TestGetLocalPeer(t *testing.T) {
|
||||
tempGlobalMinioAddr := globalMinioAddr
|
||||
defer func() {
|
||||
globalMinioAddr = tempGlobalMinioAddr
|
||||
}()
|
||||
globalMinioAddr = ":9000"
|
||||
|
||||
testCases := []struct {
|
||||
endpointArgs []string
|
||||
expectedResult string
|
||||
}{
|
||||
{[]string{"/d1", "/d2", "d3", "d4"}, ":9000"},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"},
|
||||
"localhost:9000"},
|
||||
{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"},
|
||||
"localhost:9000"},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"},
|
||||
"localhost:9000"},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
endpoints, _ := NewEndpointList(testCase.endpointArgs...)
|
||||
remotePeer := GetLocalPeer(endpoints)
|
||||
if remotePeer != testCase.expectedResult {
|
||||
t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, remotePeer)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRemotePeers(t *testing.T) {
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
|
||||
@@ -116,3 +116,77 @@ func TestErasureCreateFile(t *testing.T) {
|
||||
setup.Remove()
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmarks
|
||||
|
||||
func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
|
||||
setup, err := newErasureTestSetup(data, parity, blockSizeV1)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create test setup: %v", err)
|
||||
}
|
||||
defer setup.Remove()
|
||||
storage, err := NewErasureStorage(setup.disks, data, parity)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create ErasureStorage: %v", err)
|
||||
}
|
||||
buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
|
||||
content := make([]byte, size)
|
||||
|
||||
for i := 0; i < dataDown; i++ {
|
||||
storage.disks[i] = OfflineDisk
|
||||
}
|
||||
for i := data; i < data+parityDown; i++ {
|
||||
storage.disks[i] = OfflineDisk
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.SetBytes(size)
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkErasureWriteQuick(b *testing.B) {
|
||||
const size = 12 * 1024 * 1024
|
||||
b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
|
||||
b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
|
||||
b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureWrite_4_64KB(b *testing.B) {
|
||||
const size = 64 * 1024
|
||||
b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 0, size, b) })
|
||||
b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 0, 1, size, b) })
|
||||
b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureWrite(2, 2, 1, 0, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureWrite_8_20MB(b *testing.B) {
|
||||
const size = 20 * 1024 * 1024
|
||||
b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 0, size, b) })
|
||||
b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 1, size, b) })
|
||||
b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 1, 0, size, b) })
|
||||
b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 0, 3, size, b) })
|
||||
b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureWrite(4, 4, 3, 0, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureWrite_12_30MB(b *testing.B) {
|
||||
const size = 30 * 1024 * 1024
|
||||
b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 0, size, b) })
|
||||
b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 1, size, b) })
|
||||
b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 1, 0, size, b) })
|
||||
b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 0, 5, size, b) })
|
||||
b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureWrite(6, 6, 5, 0, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureWrite_16_40MB(b *testing.B) {
|
||||
const size = 40 * 1024 * 1024
|
||||
b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 0, size, b) })
|
||||
b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 1, size, b) })
|
||||
b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 1, 0, size, b) })
|
||||
b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 0, 7, size, b) })
|
||||
b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureWrite(8, 8, 7, 0, size, b) })
|
||||
}
|
||||
|
||||
@@ -222,3 +222,92 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
|
||||
buf.Reset()
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmarks
|
||||
|
||||
func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b *testing.B) {
|
||||
setup, err := newErasureTestSetup(data, parity, blockSizeV1)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create test setup: %v", err)
|
||||
}
|
||||
defer setup.Remove()
|
||||
storage, err := NewErasureStorage(setup.disks, data, parity)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create ErasureStorage: %v", err)
|
||||
}
|
||||
|
||||
content := make([]byte, size)
|
||||
buffer := make([]byte, blockSizeV1, 2*blockSizeV1)
|
||||
file, err := storage.CreateFile(bytes.NewReader(content), "testbucket", "object", buffer, DefaultBitrotAlgorithm, data+1)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create erasure test file: %v", err)
|
||||
}
|
||||
checksums := file.Checksums
|
||||
|
||||
for i := 0; i < dataDown; i++ {
|
||||
storage.disks[i] = OfflineDisk
|
||||
}
|
||||
for i := data; i < data+parityDown; i++ {
|
||||
storage.disks[i] = OfflineDisk
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.SetBytes(size)
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if file, err = storage.ReadFile(bytes.NewBuffer(content[:0]), "testbucket", "object", 0, size, size, checksums, DefaultBitrotAlgorithm, blockSizeV1); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkErasureReadQuick(b *testing.B) {
|
||||
const size = 12 * 1024 * 1024
|
||||
b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
|
||||
b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
|
||||
b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
|
||||
b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureRead_4_64KB(b *testing.B) {
|
||||
const size = 64 * 1024
|
||||
b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 0, size, b) })
|
||||
b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 1, size, b) })
|
||||
b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 0, size, b) })
|
||||
b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 1, 1, size, b) })
|
||||
b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureRead(2, 2, 0, 2, size, b) })
|
||||
b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureRead(2, 2, 2, 0, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureRead_8_20MB(b *testing.B) {
|
||||
const size = 20 * 1024 * 1024
|
||||
b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 0, size, b) })
|
||||
b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 1, size, b) })
|
||||
b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 0, size, b) })
|
||||
b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 1, 1, size, b) })
|
||||
b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureRead(4, 4, 0, 4, size, b) })
|
||||
b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 2, 2, size, b) })
|
||||
b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureRead(4, 4, 4, 0, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureRead_12_30MB(b *testing.B) {
|
||||
const size = 30 * 1024 * 1024
|
||||
b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 0, size, b) })
|
||||
b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 1, size, b) })
|
||||
b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 0, size, b) })
|
||||
b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 1, 1, size, b) })
|
||||
b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureRead(6, 6, 0, 6, size, b) })
|
||||
b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 3, 3, size, b) })
|
||||
b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureRead(6, 6, 6, 0, size, b) })
|
||||
}
|
||||
|
||||
func BenchmarkErasureRead_16_40MB(b *testing.B) {
|
||||
const size = 40 * 1024 * 1024
|
||||
b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 0, size, b) })
|
||||
b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 1, size, b) })
|
||||
b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 0, size, b) })
|
||||
b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 1, 1, size, b) })
|
||||
b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureRead(8, 8, 0, 8, size, b) })
|
||||
b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 4, 4, size, b) })
|
||||
b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureRead(8, 8, 8, 0, size, b) })
|
||||
}
|
||||
|
||||
+48
-26
@@ -27,6 +27,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -59,7 +60,7 @@ type internalNotifier struct {
|
||||
// Connected listeners is a map of listener ARNs to channels
|
||||
// on which the ListenBucket API handler go routine is waiting
|
||||
// for events to send to a client.
|
||||
connectedListeners map[string]chan []NotificationEvent
|
||||
connectedListeners map[string]*listenChan
|
||||
|
||||
rwMutex *sync.RWMutex
|
||||
}
|
||||
@@ -166,13 +167,13 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
|
||||
// For all other events we should set ETag and Size.
|
||||
nEvent.S3.Object = objectMeta{
|
||||
Key: escapedObj,
|
||||
ETag: event.ObjInfo.ETag,
|
||||
Size: event.ObjInfo.Size,
|
||||
ContentType: event.ObjInfo.ContentType,
|
||||
UserDefined: event.ObjInfo.UserDefined,
|
||||
VersionID: "1",
|
||||
Sequencer: uniqueID,
|
||||
Key: escapedObj,
|
||||
ETag: event.ObjInfo.ETag,
|
||||
Size: event.ObjInfo.Size,
|
||||
ContentType: event.ObjInfo.ContentType,
|
||||
UserMetadata: event.ObjInfo.UserDefined,
|
||||
VersionID: "1",
|
||||
Sequencer: uniqueID,
|
||||
}
|
||||
|
||||
// Success.
|
||||
@@ -205,7 +206,7 @@ func (en eventNotifier) GetInternalTarget(arn string) *listenerLogger {
|
||||
}
|
||||
|
||||
// Set a new sns target for an input sns ARN.
|
||||
func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh chan []NotificationEvent) error {
|
||||
func (en *eventNotifier) AddListenerChan(snsARN string, listenerCh *listenChan) error {
|
||||
if listenerCh == nil {
|
||||
return errInvalidArgument
|
||||
}
|
||||
@@ -228,9 +229,9 @@ func (en *eventNotifier) SendListenerEvent(arn string, event []NotificationEvent
|
||||
en.internal.rwMutex.Lock()
|
||||
defer en.internal.rwMutex.Unlock()
|
||||
|
||||
ch, ok := en.internal.connectedListeners[arn]
|
||||
listenChan, ok := en.internal.connectedListeners[arn]
|
||||
if ok {
|
||||
ch <- event
|
||||
listenChan.sendNotificationEvent(event)
|
||||
}
|
||||
// If the channel is not present we ignore the event.
|
||||
return nil
|
||||
@@ -377,18 +378,24 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
|
||||
// 'errNoSuchNotifications'. This is default when no
|
||||
// bucket notifications are found on the bucket.
|
||||
if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
|
||||
return nil, errNoSuchNotifications
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
}
|
||||
errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
|
||||
// Returns error for other errors.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// if `notifications.xml` is empty we should return NoSuchNotifications.
|
||||
if buffer.Len() == 0 {
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
}
|
||||
|
||||
// Unmarshal notification bytes.
|
||||
notificationConfigBytes := buffer.Bytes()
|
||||
notificationCfg := ¬ificationConfig{}
|
||||
if err = xml.Unmarshal(notificationConfigBytes, ¬ificationCfg); err != nil {
|
||||
return nil, err
|
||||
// Unmarshal notification bytes only if we read data.
|
||||
if err = xml.Unmarshal(notificationConfigBytes, notificationCfg); err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
|
||||
// Return success.
|
||||
@@ -418,23 +425,27 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
|
||||
var buffer bytes.Buffer
|
||||
err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer)
|
||||
if err != nil {
|
||||
// 'notification.xml' not found return
|
||||
// 'listener.json' not found return
|
||||
// 'errNoSuchNotifications'. This is default when no
|
||||
// bucket listeners are found on the bucket.
|
||||
if isErrObjectNotFound(err) {
|
||||
return nil, errNoSuchNotifications
|
||||
// bucket listeners are found on the bucket
|
||||
if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
}
|
||||
errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
|
||||
// Returns error for other errors.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Unmarshal notification bytes.
|
||||
// if `listener.json` is empty we should return NoSuchNotifications.
|
||||
if buffer.Len() == 0 {
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
}
|
||||
|
||||
var lCfg []listenerConfig
|
||||
lConfigBytes := buffer.Bytes()
|
||||
if err = json.Unmarshal(lConfigBytes, &lCfg); err != nil {
|
||||
errorIf(err, "Unable to unmarshal listener config from JSON.")
|
||||
return nil, err
|
||||
return nil, traceError(err)
|
||||
}
|
||||
|
||||
// Return success.
|
||||
@@ -459,8 +470,12 @@ func persistNotificationConfig(bucket string, ncfg *notificationConfig, obj Obje
|
||||
defer objLock.Unlock()
|
||||
|
||||
// write object to path
|
||||
sha256Sum := getSHA256Hash(buf)
|
||||
_, err = obj.PutObject(minioMetaBucket, ncPath, NewHashReader(bytes.NewReader(buf), int64(len(buf)), "", sha256Sum), nil)
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to write bucket notification configuration.")
|
||||
return err
|
||||
}
|
||||
_, err = obj.PutObject(minioMetaBucket, ncPath, hashReader, nil)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to write bucket notification configuration.")
|
||||
return err
|
||||
@@ -486,12 +501,19 @@ func persistListenerConfig(bucket string, lcfg []listenerConfig, obj ObjectLayer
|
||||
defer objLock.Unlock()
|
||||
|
||||
// write object to path
|
||||
sha256Sum := getSHA256Hash(buf)
|
||||
_, err = obj.PutObject(minioMetaBucket, lcPath, NewHashReader(bytes.NewReader(buf), int64(len(buf)), "", sha256Sum), nil)
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to write bucket listener configuration to object layer.")
|
||||
return err
|
||||
}
|
||||
return err
|
||||
|
||||
// write object to path
|
||||
_, err = obj.PutObject(minioMetaBucket, lcPath, hashReader, nil)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to write bucket listener configuration to object layer.")
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Removes notification.xml for a given bucket, only used during DeleteBucket.
|
||||
@@ -811,7 +833,7 @@ func initEventNotifier(objAPI ObjectLayer) error {
|
||||
rwMutex: &sync.RWMutex{},
|
||||
targets: listenTargets,
|
||||
listenerConfigs: lConfigs,
|
||||
connectedListeners: make(map[string]chan []NotificationEvent),
|
||||
connectedListeners: make(map[string]*listenChan),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -62,7 +62,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
|
||||
notificationXML += "</NotificationConfiguration>"
|
||||
size := int64(len([]byte(notificationXML)))
|
||||
reader := bytes.NewReader([]byte(notificationXML))
|
||||
if _, err := xl.PutObject(minioMetaBucket, bucketConfigPrefix+"/"+bucketName+"/"+bucketNotificationConfig, NewHashReader(reader, size, "", ""), nil); err != nil {
|
||||
if _, err := xl.PutObject(minioMetaBucket, bucketConfigPrefix+"/"+bucketName+"/"+bucketNotificationConfig, mustGetHashReader(t, reader, size, "", ""), nil); err != nil {
|
||||
t.Fatal("Unexpected error:", err)
|
||||
}
|
||||
|
||||
@@ -462,11 +462,9 @@ func TestListenBucketNotification(t *testing.T) {
|
||||
}
|
||||
|
||||
// Create a new notification event channel.
|
||||
nEventCh := make(chan []NotificationEvent)
|
||||
// Close the listener channel.
|
||||
defer close(nEventCh)
|
||||
nListenCh := newListenChan()
|
||||
// Add events channel for listener.
|
||||
if err := globalEventNotifier.AddListenerChan(listenARN, nEventCh); err != nil {
|
||||
if err := globalEventNotifier.AddListenerChan(listenARN, nListenCh); err != nil {
|
||||
t.Fatalf("Test Setup error: %v", err)
|
||||
}
|
||||
// Remove listen channel after the writer has closed or the
|
||||
@@ -489,7 +487,7 @@ func TestListenBucketNotification(t *testing.T) {
|
||||
// Wait for the event notification here, if nothing is received within 30 seconds,
|
||||
// test error will be fired
|
||||
select {
|
||||
case n := <-nEventCh:
|
||||
case n := <-nListenCh.dataCh:
|
||||
// Check that received event
|
||||
if len(n) == 0 {
|
||||
t.Fatal("Unexpected error occurred")
|
||||
@@ -497,9 +495,7 @@ func TestListenBucketNotification(t *testing.T) {
|
||||
if n[0].S3.Object.Key != objectName {
|
||||
t.Fatalf("Received wrong object name in notification, expected %s, received %s", n[0].S3.Object.Key, objectName)
|
||||
}
|
||||
break
|
||||
case <-time.After(3 * time.Second):
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -531,30 +531,6 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
|
||||
return format, nil
|
||||
}
|
||||
|
||||
// isFormatNotFound - returns true if all `format.json` are not found on all disks.
|
||||
func isFormatNotFound(formats []*formatConfigV1) bool {
|
||||
for _, format := range formats {
|
||||
// One of the `format.json` is found.
|
||||
if format != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// All format.json missing, success.
|
||||
return true
|
||||
}
|
||||
|
||||
// isFormatFound - returns true if all input formats are found on all disks.
|
||||
func isFormatFound(formats []*formatConfigV1) bool {
|
||||
for _, format := range formats {
|
||||
// One of `format.json` is not found.
|
||||
if format == nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// All format.json present, success.
|
||||
return true
|
||||
}
|
||||
|
||||
// collectNSaveNewFormatConfigs - creates new format configs based on
|
||||
// the reference config and saves it on all disks, this is to be
|
||||
// called from healFormatXL* functions.
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
@@ -236,11 +237,15 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
|
||||
bucket := "bucket"
|
||||
object := "object"
|
||||
|
||||
_, err = obj.PutObject(bucket, object, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
hashReader, err := hash.NewReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", "")
|
||||
if err != nil {
|
||||
return []StorageAPI{}, err
|
||||
}
|
||||
|
||||
if _, err = obj.PutObject(bucket, object, hashReader, nil); err != nil {
|
||||
return []StorageAPI{}, err
|
||||
}
|
||||
|
||||
// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
|
||||
// created when minio starts
|
||||
for i := 3; i <= 5; i++ {
|
||||
@@ -326,7 +331,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
|
||||
bucket := "bucket"
|
||||
object := "object"
|
||||
|
||||
_, err = obj.PutObject(bucket, object, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
_, err = obj.PutObject(bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -400,7 +405,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
|
||||
bucket := "bucket"
|
||||
object := "object"
|
||||
|
||||
_, err = obj.PutObject(bucket, object, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
_, err = obj.PutObject(bucket, object, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+70
-52
@@ -37,12 +37,7 @@ func fsRemoveFile(filePath string) (err error) {
|
||||
}
|
||||
|
||||
if err = os.Remove((filePath)); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return traceError(errFileNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
return traceError(errFileAccessDenied)
|
||||
}
|
||||
return traceError(err)
|
||||
return osErrToFSFileErr(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -94,6 +89,24 @@ func fsRemoveDir(dirPath string) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Creates a new directory, parent dir is also recursively created
|
||||
// if it doesn't exist.
|
||||
func fsMkdirAll(dirPath string) (err error) {
|
||||
if dirPath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err = checkPathLength(dirPath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(dirPath, 0777); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Creates a new directory, parent dir should exist
|
||||
// otherwise returns an error. If directory already
|
||||
// exists returns an error. Windows long paths
|
||||
@@ -126,6 +139,11 @@ func fsMkdir(dirPath string) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// fsStat is a low level call which validates input arguments
|
||||
// and checks input length upto supported maximum. Does
|
||||
// not perform any higher layer interpretation of files v/s
|
||||
// directories. For higher level interpretation look at
|
||||
// fsStatFileDir, fsStatFile, fsStatDir.
|
||||
func fsStat(statLoc string) (os.FileInfo, error) {
|
||||
if statLoc == "" {
|
||||
return nil, traceError(errInvalidArgument)
|
||||
@@ -133,7 +151,7 @@ func fsStat(statLoc string) (os.FileInfo, error) {
|
||||
if err := checkPathLength(statLoc); err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
fi, err := osStat((statLoc))
|
||||
fi, err := os.Stat((statLoc))
|
||||
if err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
@@ -141,10 +159,9 @@ func fsStat(statLoc string) (os.FileInfo, error) {
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// Lookup if directory exists, returns directory
|
||||
// attributes upon success.
|
||||
func fsStatDir(statDir string) (os.FileInfo, error) {
|
||||
fi, err := fsStat(statDir)
|
||||
// Lookup if volume exists, returns volume attributes upon success.
|
||||
func fsStatVolume(volume string) (os.FileInfo, error) {
|
||||
fi, err := fsStat(volume)
|
||||
if err != nil {
|
||||
err = errorCause(err)
|
||||
if os.IsNotExist(err) {
|
||||
@@ -162,21 +179,47 @@ func fsStatDir(statDir string) (os.FileInfo, error) {
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// Lookup if file exists, returns file attributes upon success
|
||||
// Is a one place function which converts all os.PathError
|
||||
// into a more FS object layer friendly form, converts
|
||||
// known errors into their typed form for top level
|
||||
// interpretation.
|
||||
func osErrToFSFileErr(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
err = errorCause(err)
|
||||
if os.IsNotExist(err) {
|
||||
return traceError(errFileNotFound)
|
||||
}
|
||||
if os.IsPermission(err) {
|
||||
return traceError(errFileAccessDenied)
|
||||
}
|
||||
if isSysErrNotDir(err) {
|
||||
return traceError(errFileAccessDenied)
|
||||
}
|
||||
if isSysErrPathNotFound(err) {
|
||||
return traceError(errFileNotFound)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Lookup if directory exists, returns directory attributes upon success.
|
||||
func fsStatDir(statDir string) (os.FileInfo, error) {
|
||||
fi, err := fsStat(statDir)
|
||||
if err != nil {
|
||||
return nil, osErrToFSFileErr(err)
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return nil, traceError(errFileAccessDenied)
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// Lookup if file exists, returns file attributes upon success.
|
||||
func fsStatFile(statFile string) (os.FileInfo, error) {
|
||||
fi, err := fsStat(statFile)
|
||||
if err != nil {
|
||||
err = errorCause(err)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, traceError(errFileNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
return nil, traceError(errFileAccessDenied)
|
||||
} else if isSysErrNotDir(err) {
|
||||
return nil, traceError(errFileAccessDenied)
|
||||
} else if isSysErrPathNotFound(err) {
|
||||
return nil, traceError(errFileNotFound)
|
||||
}
|
||||
return nil, traceError(err)
|
||||
return nil, osErrToFSFileErr(err)
|
||||
}
|
||||
if fi.IsDir() {
|
||||
return nil, traceError(errFileAccessDenied)
|
||||
@@ -196,22 +239,11 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
|
||||
|
||||
fr, err := os.Open((readPath))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, 0, traceError(errFileNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
return nil, 0, traceError(errFileAccessDenied)
|
||||
} else if isSysErrNotDir(err) {
|
||||
// File path cannot be verified since one of the parents is a file.
|
||||
return nil, 0, traceError(errFileAccessDenied)
|
||||
} else if isSysErrPathNotFound(err) {
|
||||
// Add specific case for windows.
|
||||
return nil, 0, traceError(errFileNotFound)
|
||||
}
|
||||
return nil, 0, traceError(err)
|
||||
return nil, 0, osErrToFSFileErr(err)
|
||||
}
|
||||
|
||||
// Stat to get the size of the file at path.
|
||||
st, err := osStat((readPath))
|
||||
st, err := os.Stat((readPath))
|
||||
if err != nil {
|
||||
return nil, 0, traceError(err)
|
||||
}
|
||||
@@ -253,11 +285,7 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
|
||||
|
||||
writer, err := os.OpenFile((filePath), os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
// File path cannot be verified since one of the parents is a file.
|
||||
if isSysErrNotDir(err) {
|
||||
return 0, traceError(errFileAccessDenied)
|
||||
}
|
||||
return 0, err
|
||||
return 0, osErrToFSFileErr(err)
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
@@ -346,17 +374,7 @@ func fsRenameFile(sourcePath, destPath string) error {
|
||||
}
|
||||
// Verify if source path exists.
|
||||
if _, err := os.Stat((sourcePath)); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return traceError(errFileNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
return traceError(errFileAccessDenied)
|
||||
} else if isSysErrPathNotFound(err) {
|
||||
return traceError(errFileNotFound)
|
||||
} else if isSysErrNotDir(err) {
|
||||
// File path cannot be verified since one of the parents is a file.
|
||||
return traceError(errFileAccessDenied)
|
||||
}
|
||||
return traceError(err)
|
||||
return osErrToFSFileErr(err)
|
||||
}
|
||||
if err := os.MkdirAll(pathutil.Dir(destPath), 0777); err != nil {
|
||||
return traceError(err)
|
||||
|
||||
@@ -27,6 +27,28 @@ import (
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// Tests - fsMkdirAll()
|
||||
func TestFSMkdirAll(t *testing.T) {
|
||||
// create posix test setup
|
||||
_, path, err := newPosixTestSetup()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create posix test setup, %s", err)
|
||||
}
|
||||
defer os.RemoveAll(path)
|
||||
|
||||
if err = fsMkdirAll(""); errorCause(err) != errInvalidArgument {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if err = fsMkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if err = fsMkdirAll(pathJoin(path, "success-vol", "success-object")); err != nil {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSRenameFile(t *testing.T) {
|
||||
// create posix test setup
|
||||
_, path, err := newPosixTestSetup()
|
||||
@@ -173,7 +195,7 @@ func TestFSStats(t *testing.T) {
|
||||
t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
} else {
|
||||
if _, err := fsStatDir(pathJoin(testCase.srcFSPath, testCase.srcVol)); errorCause(err) != testCase.expectedErr {
|
||||
if _, err := fsStatVolume(pathJoin(testCase.srcFSPath, testCase.srcVol)); errorCause(err) != testCase.expectedErr {
|
||||
t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
@@ -538,11 +560,11 @@ func TestFSRemoveMeta(t *testing.T) {
|
||||
t.Fatalf("Unable to remove file, %s", err)
|
||||
}
|
||||
|
||||
if _, err := osStat((filePath)); !os.IsNotExist(err) {
|
||||
if _, err := os.Stat((filePath)); !os.IsNotExist(err) {
|
||||
t.Fatalf("`%s` file found though it should have been deleted.", filePath)
|
||||
}
|
||||
|
||||
if _, err := osStat((path.Dir(filePath))); !os.IsNotExist(err) {
|
||||
if _, err := os.Stat((path.Dir(filePath))); !os.IsNotExist(err) {
|
||||
t.Fatalf("`%s` parent directory found though it should have been deleted.", filePath)
|
||||
}
|
||||
}
|
||||
|
||||
+16
-7
@@ -91,6 +91,11 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
|
||||
}
|
||||
}
|
||||
|
||||
if hasSuffix(object, slashSeparator) {
|
||||
m.Meta["etag"] = emptyETag // For directories etag is d41d8cd98f00b204e9800998ecf8427e
|
||||
m.Meta["content-type"] = "application/octet-stream"
|
||||
}
|
||||
|
||||
objInfo := ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
@@ -101,7 +106,11 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
|
||||
if fi != nil {
|
||||
objInfo.ModTime = fi.ModTime()
|
||||
objInfo.Size = fi.Size()
|
||||
objInfo.IsDir = fi.IsDir()
|
||||
if fi.IsDir() {
|
||||
// Directory is always 0 bytes in S3 API, treat it as such.
|
||||
objInfo.Size = 0
|
||||
objInfo.IsDir = fi.IsDir()
|
||||
}
|
||||
}
|
||||
|
||||
// Extract etag from metadata.
|
||||
@@ -350,15 +359,15 @@ func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Return if the part info in uploadedParts and completeParts are same.
|
||||
func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) bool {
|
||||
if len(uploadedParts) != len(completeParts) {
|
||||
// Return if the part info in uploadedParts and CompleteParts are same.
|
||||
func isPartsSame(uploadedParts []objectPartInfo, CompleteParts []CompletePart) bool {
|
||||
if len(uploadedParts) != len(CompleteParts) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range completeParts {
|
||||
if uploadedParts[i].Number != completeParts[i].PartNumber ||
|
||||
uploadedParts[i].ETag != completeParts[i].ETag {
|
||||
for i := range CompleteParts {
|
||||
if uploadedParts[i].Number != CompleteParts[i].PartNumber ||
|
||||
uploadedParts[i].ETag != CompleteParts[i].ETag {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ func TestReadFSMetadata(t *testing.T) {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
if _, err := obj.PutObject(bucketName, objectName, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
|
||||
if _, err := obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
@@ -87,7 +87,7 @@ func TestWriteFSMetadata(t *testing.T) {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
if _, err := obj.PutObject(bucketName, objectName, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
|
||||
if _, err := obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
|
||||
+175
-70
@@ -25,6 +25,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
@@ -109,8 +110,8 @@ func (fs fsObjects) addUploadID(bucket, object, uploadID string, initiated time.
|
||||
}
|
||||
|
||||
// listMultipartUploadIDs - list all the upload ids from a marker up to 'count'.
|
||||
func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int) ([]uploadMetadata, bool, error) {
|
||||
var uploads []uploadMetadata
|
||||
func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarker string, count int) ([]MultipartInfo, bool, error) {
|
||||
var uploads []MultipartInfo
|
||||
|
||||
// Hold the lock so that two parallel complete-multipart-uploads
|
||||
// do not leave a stale uploads.json behind.
|
||||
@@ -148,7 +149,7 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
|
||||
}
|
||||
|
||||
for index < len(uploadIDs.Uploads) {
|
||||
uploads = append(uploads, uploadMetadata{
|
||||
uploads = append(uploads, MultipartInfo{
|
||||
Object: objectName,
|
||||
UploadID: uploadIDs.Uploads[index].UploadID,
|
||||
Initiated: uploadIDs.Uploads[index].Initiated,
|
||||
@@ -164,8 +165,8 @@ func (fs fsObjects) listMultipartUploadIDs(bucketName, objectName, uploadIDMarke
|
||||
return uploads, end, nil
|
||||
}
|
||||
|
||||
// listMultipartUploads - lists all multipart uploads.
|
||||
func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
// listMultipartUploadsCleanup - lists all multipart uploads. Called by fs.cleanupStaleMultipartUpload()
|
||||
func (fs fsObjects) listMultipartUploadsCleanup(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
result := ListMultipartsInfo{}
|
||||
recursive := true
|
||||
if delimiter == slashSeparator {
|
||||
@@ -190,7 +191,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
|
||||
multipartMarkerPath = pathJoin(bucket, keyMarker)
|
||||
}
|
||||
|
||||
var uploads []uploadMetadata
|
||||
var uploads []MultipartInfo
|
||||
var err error
|
||||
var eof bool
|
||||
|
||||
@@ -243,7 +244,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
|
||||
|
||||
entry := strings.TrimPrefix(walkResult.entry, retainSlash(bucket))
|
||||
if hasSuffix(walkResult.entry, slashSeparator) {
|
||||
uploads = append(uploads, uploadMetadata{
|
||||
uploads = append(uploads, MultipartInfo{
|
||||
Object: entry,
|
||||
})
|
||||
maxUploads--
|
||||
@@ -256,7 +257,7 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
|
||||
continue
|
||||
}
|
||||
|
||||
var tmpUploads []uploadMetadata
|
||||
var tmpUploads []MultipartInfo
|
||||
var end bool
|
||||
uploadIDMarker = ""
|
||||
|
||||
@@ -308,36 +309,10 @@ func (fs fsObjects) listMultipartUploads(bucket, prefix, keyMarker, uploadIDMark
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListMultipartUploads - returns empty response always. The onus is
|
||||
// on applications to remember uploadId of the multipart uploads that
|
||||
// are in progress.
|
||||
func (fs fsObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
|
||||
return lmi, err
|
||||
}
|
||||
|
||||
if _, err := fs.statBucketDir(bucket); err != nil {
|
||||
return lmi, toObjectErr(err, bucket)
|
||||
}
|
||||
return ListMultipartsInfo{
|
||||
Prefix: prefix,
|
||||
KeyMarker: keyMarker,
|
||||
UploadIDMarker: uploadIDMarker,
|
||||
Delimiter: delimiter,
|
||||
MaxUploads: maxUploads,
|
||||
IsTruncated: false,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// listMultipartUploadsHelper - lists all the pending multipart uploads on a
|
||||
// bucket. Additionally takes 'prefix, keyMarker, uploadIDmarker and a
|
||||
// delimiter' which allows us to list uploads match a particular
|
||||
// prefix or lexically starting from 'keyMarker' or delimiting the
|
||||
// output to get a directory like listing.
|
||||
//
|
||||
// This function remains here to aid in listing uploads that require cleanup.
|
||||
func (fs fsObjects) listMultipartUploadsHelper(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
if err := checkListMultipartArgs(bucket, prefix, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
|
||||
// ListMultipartUploads - lists all the uploadIDs for the specified object.
|
||||
// We do not support prefix based listing.
|
||||
func (fs fsObjects) ListMultipartUploads(bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
if err := checkListMultipartArgs(bucket, object, keyMarker, uploadIDMarker, delimiter, fs); err != nil {
|
||||
return lmi, err
|
||||
}
|
||||
|
||||
@@ -345,7 +320,35 @@ func (fs fsObjects) listMultipartUploadsHelper(bucket, prefix, keyMarker, upload
|
||||
return lmi, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
return fs.listMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
result := ListMultipartsInfo{}
|
||||
|
||||
result.IsTruncated = true
|
||||
result.MaxUploads = maxUploads
|
||||
result.KeyMarker = keyMarker
|
||||
result.Prefix = object
|
||||
result.Delimiter = delimiter
|
||||
|
||||
uploads, _, err := fs.listMultipartUploadIDs(bucket, object, uploadIDMarker, maxUploads)
|
||||
if err != nil {
|
||||
return lmi, err
|
||||
}
|
||||
|
||||
result.NextKeyMarker = object
|
||||
// Loop through all the received uploads fill in the multiparts result.
|
||||
for _, upload := range uploads {
|
||||
uploadID := upload.UploadID
|
||||
result.Uploads = append(result.Uploads, upload)
|
||||
result.NextUploadIDMarker = uploadID
|
||||
}
|
||||
|
||||
result.IsTruncated = len(uploads) == maxUploads
|
||||
|
||||
if !result.IsTruncated {
|
||||
result.NextKeyMarker = ""
|
||||
result.NextUploadIDMarker = ""
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// newMultipartUpload - wrapper for initializing a new multipart
|
||||
@@ -456,7 +459,12 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
|
||||
pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
|
||||
}()
|
||||
|
||||
partInfo, err := fs.PutObjectPart(dstBucket, dstObject, uploadID, partID, NewHashReader(pipeReader, length, "", ""))
|
||||
hashReader, err := hash.NewReader(pipeReader, length, "", "")
|
||||
if err != nil {
|
||||
return pi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
|
||||
partInfo, err := fs.PutObjectPart(dstBucket, dstObject, uploadID, partID, hashReader)
|
||||
if err != nil {
|
||||
return pi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
@@ -471,7 +479,7 @@ func (fs fsObjects) CopyObjectPart(srcBucket, srcObject, dstBucket, dstObject, u
|
||||
// an ongoing multipart transaction. Internally incoming data is
|
||||
// written to '.minio.sys/tmp' location and safely renamed to
|
||||
// '.minio.sys/multipart' for reach parts.
|
||||
func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *HashReader) (pi PartInfo, e error) {
|
||||
func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) {
|
||||
if err := checkPutObjectPartArgs(bucket, object, fs); err != nil {
|
||||
return pi, err
|
||||
}
|
||||
@@ -480,6 +488,11 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
|
||||
return pi, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
// Validate input data size and it can never be less than zero.
|
||||
if data.Size() < 0 {
|
||||
return pi, toObjectErr(traceError(errInvalidArgument))
|
||||
}
|
||||
|
||||
// Hold the lock so that two parallel complete-multipart-uploads
|
||||
// do not leave a stale uploads.json behind.
|
||||
objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
|
||||
@@ -545,10 +558,6 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
|
||||
// delete.
|
||||
defer fsRemoveFile(fsPartPath)
|
||||
|
||||
if err = data.Verify(); err != nil {
|
||||
return pi, toObjectErr(err, minioMetaTmpBucket, tmpPartPath)
|
||||
}
|
||||
|
||||
partPath := pathJoin(bucket, object, uploadID, partSuffix)
|
||||
// Lock the part so that another part upload with same part-number gets blocked
|
||||
// while the part is getting appended in the background.
|
||||
@@ -563,9 +572,10 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
|
||||
return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
|
||||
}
|
||||
|
||||
md5hex := hex.EncodeToString(data.MD5Current())
|
||||
|
||||
// Save the object part info in `fs.json`.
|
||||
md5Hex := hex.EncodeToString(data.MD5())
|
||||
fsMeta.AddObjectPart(partID, partSuffix, md5Hex, data.Size())
|
||||
fsMeta.AddObjectPart(partID, partSuffix, md5hex, data.Size())
|
||||
if _, err = fsMeta.WriteTo(rwlk); err != nil {
|
||||
partLock.Unlock()
|
||||
return pi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
|
||||
@@ -591,14 +601,86 @@ func (fs fsObjects) PutObjectPart(bucket, object, uploadID string, partID int, d
|
||||
return PartInfo{
|
||||
PartNumber: partID,
|
||||
LastModified: fi.ModTime(),
|
||||
ETag: md5Hex,
|
||||
ETag: md5hex,
|
||||
Size: fi.Size(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ListObjectParts - returns empty response always. Applications are
|
||||
// expected to remember the uploaded part numbers and corresponding
|
||||
// etags.
|
||||
// listObjectParts - wrapper scanning through
|
||||
// '.minio.sys/multipart/bucket/object/UPLOADID'. Lists all the parts
|
||||
// saved inside '.minio.sys/multipart/bucket/object/UPLOADID'.
|
||||
func (fs fsObjects) listObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi ListPartsInfo, e error) {
|
||||
result := ListPartsInfo{}
|
||||
|
||||
uploadIDPath := pathJoin(bucket, object, uploadID)
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, fsMetaJSONFile)
|
||||
metaFile, err := fs.rwPool.Open(fsMetaPath)
|
||||
if err != nil {
|
||||
if err == errFileNotFound || err == errFileAccessDenied {
|
||||
// On windows oddly this is returned.
|
||||
return lpi, traceError(InvalidUploadID{UploadID: uploadID})
|
||||
}
|
||||
return lpi, toObjectErr(traceError(err), bucket, object)
|
||||
}
|
||||
defer fs.rwPool.Close(fsMetaPath)
|
||||
|
||||
fsMeta := fsMetaV1{}
|
||||
_, err = fsMeta.ReadFrom(metaFile.LockedFile)
|
||||
if err != nil {
|
||||
return lpi, toObjectErr(err, minioMetaBucket, fsMetaPath)
|
||||
}
|
||||
|
||||
// Only parts with higher part numbers will be listed.
|
||||
partIdx := fsMeta.ObjectPartIndex(partNumberMarker)
|
||||
parts := fsMeta.Parts
|
||||
if partIdx != -1 {
|
||||
parts = fsMeta.Parts[partIdx+1:]
|
||||
}
|
||||
|
||||
count := maxParts
|
||||
for _, part := range parts {
|
||||
var fi os.FileInfo
|
||||
partNamePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, uploadIDPath, part.Name)
|
||||
fi, err = fsStatFile(partNamePath)
|
||||
if err != nil {
|
||||
return lpi, toObjectErr(err, minioMetaMultipartBucket, partNamePath)
|
||||
}
|
||||
result.Parts = append(result.Parts, PartInfo{
|
||||
PartNumber: part.Number,
|
||||
ETag: part.ETag,
|
||||
LastModified: fi.ModTime(),
|
||||
Size: fi.Size(),
|
||||
})
|
||||
count--
|
||||
if count == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// If listed entries are more than maxParts, we set IsTruncated as true.
|
||||
if len(parts) > len(result.Parts) {
|
||||
result.IsTruncated = true
|
||||
// Make sure to fill next part number marker if IsTruncated is
|
||||
// true for subsequent listing.
|
||||
nextPartNumberMarker := result.Parts[len(result.Parts)-1].PartNumber
|
||||
result.NextPartNumberMarker = nextPartNumberMarker
|
||||
}
|
||||
result.Bucket = bucket
|
||||
result.Object = object
|
||||
result.UploadID = uploadID
|
||||
result.MaxParts = maxParts
|
||||
|
||||
// Success.
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// ListObjectParts - lists all previously uploaded parts for a given
|
||||
// object and uploadID. Takes additional input of part-number-marker
|
||||
// to indicate where the listing should begin from.
|
||||
//
|
||||
// Implements S3 compatible ListObjectParts API. The resulting
|
||||
// ListPartsInfo structure is unmarshalled directly into XML and
|
||||
// replied back to the client.
|
||||
func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi ListPartsInfo, e error) {
|
||||
if err := checkListPartsArgs(bucket, object, fs); err != nil {
|
||||
return lpi, err
|
||||
@@ -609,24 +691,20 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
|
||||
return lpi, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
// Check if uploadID exists. N B This check may race with a
|
||||
// concurrent abortMultipartUpload on the same uploadID. It is
|
||||
// OK to be eventually consistent w.r.t listing of objects,
|
||||
// uploads and parts.
|
||||
uploadIDPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object, uploadID)
|
||||
_, err := fsStatDir(uploadIDPath)
|
||||
// Hold the lock so that two parallel complete-multipart-uploads
|
||||
// do not leave a stale uploads.json behind.
|
||||
objectMPartPathLock := globalNSMutex.NewNSLock(minioMetaMultipartBucket, pathJoin(bucket, object))
|
||||
if err := objectMPartPathLock.GetRLock(globalListingTimeout); err != nil {
|
||||
return lpi, traceError(err)
|
||||
}
|
||||
defer objectMPartPathLock.RUnlock()
|
||||
|
||||
listPartsInfo, err := fs.listObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
return lpi, traceError(InvalidUploadID{UploadID: uploadID})
|
||||
return lpi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
// Return empty list parts response
|
||||
return ListPartsInfo{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
UploadID: uploadID,
|
||||
PartNumberMarker: 0,
|
||||
MaxParts: maxParts,
|
||||
}, nil
|
||||
return listPartsInfo, nil
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - completes an ongoing multipart
|
||||
@@ -635,7 +713,7 @@ func (fs fsObjects) ListObjectParts(bucket, object, uploadID string, partNumberM
|
||||
// md5sums of all the parts.
|
||||
//
|
||||
// Implements S3 compatible Complete multipart API.
|
||||
func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []completePart) (oi ObjectInfo, e error) {
|
||||
func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) {
|
||||
if err := checkCompleteMultipartArgs(bucket, object, fs); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
@@ -669,7 +747,13 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
|
||||
if removeObjectDir {
|
||||
basePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket)
|
||||
derr := fsDeleteFile(basePath, pathJoin(basePath, object))
|
||||
errorIf(derr, "unable to remove %s in %s", pathJoin(basePath, object), basePath)
|
||||
if derr = errorCause(derr); derr != nil {
|
||||
// In parallel execution, CompleteMultipartUpload could have deleted temporary
|
||||
// state files/directory, it is safe to ignore errFileNotFound
|
||||
if derr != errFileNotFound {
|
||||
errorIf(derr, "unable to remove %s in %s", pathJoin(basePath, object), basePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
objectMPartPathLock.Unlock()
|
||||
}()
|
||||
@@ -702,6 +786,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
|
||||
return oi, toObjectErr(err, minioMetaMultipartBucket, fsMetaPathMultipart)
|
||||
}
|
||||
|
||||
partSize := int64(-1) // Used later to ensure that all parts sizes are same.
|
||||
// Validate all parts and then commit to disk.
|
||||
for i, part := range parts {
|
||||
partIdx := fsMeta.ObjectPartIndex(part.PartNumber)
|
||||
@@ -724,6 +809,20 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
|
||||
PartETag: part.ETag,
|
||||
})
|
||||
}
|
||||
if partSize == -1 {
|
||||
partSize = fsMeta.Parts[partIdx].Size
|
||||
}
|
||||
// TODO: Make necessary changes in future as explained in the below comment.
|
||||
// All parts except the last part has to be of same size. We are introducing this
|
||||
// check to see if any clients break. If clients do not break then we can optimize
|
||||
// multipart PutObjectPart by writing the part at the right offset using pwrite()
|
||||
// so that we don't need to do background append at all. i.e by the time we get
|
||||
// CompleteMultipartUpload we already have the full file available which can be
|
||||
// renamed to the main name-space.
|
||||
if (i < len(parts)-1) && partSize != fsMeta.Parts[partIdx].Size {
|
||||
fs.rwPool.Close(fsMetaPathMultipart)
|
||||
return oi, traceError(PartsSizeUnequal{})
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for any competing PutObject() operation on bucket/object, since same namespace
|
||||
@@ -889,7 +988,13 @@ func (fs fsObjects) AbortMultipartUpload(bucket, object, uploadID string) error
|
||||
if removeObjectDir {
|
||||
basePath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket)
|
||||
derr := fsDeleteFile(basePath, pathJoin(basePath, object))
|
||||
errorIf(derr, "unable to remove %s in %s", pathJoin(basePath, object), basePath)
|
||||
if derr = errorCause(derr); derr != nil {
|
||||
// In parallel execution, AbortMultipartUpload could have deleted temporary
|
||||
// state files/directory, it is safe to ignore errFileNotFound
|
||||
if derr != errFileNotFound {
|
||||
errorIf(derr, "unable to remove %s in %s", pathJoin(basePath, object), basePath)
|
||||
}
|
||||
}
|
||||
}
|
||||
objectMPartPathLock.Unlock()
|
||||
}()
|
||||
@@ -944,7 +1049,7 @@ func (fs fsObjects) cleanupStaleMultipartUpload(bucket string, expiry time.Durat
|
||||
for {
|
||||
// List multipart uploads in a bucket 1000 at a time
|
||||
prefix := ""
|
||||
lmi, err = fs.listMultipartUploadsHelper(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, "", 1000)
|
||||
lmi, err = fs.listMultipartUploadsCleanup(bucket, prefix, lmi.KeyMarker, lmi.UploadIDMarker, "", 1000)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list uploads")
|
||||
return err
|
||||
|
||||
@@ -183,7 +183,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
|
||||
sha256sum := ""
|
||||
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
_, err = fs.PutObjectPart(bucketName, objectName, uploadID, 1, NewHashReader(bytes.NewReader(data), dataLen, md5Hex, sha256sum))
|
||||
_, err = fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum))
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
@@ -212,11 +212,11 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
|
||||
|
||||
md5Hex := getMD5Hash(data)
|
||||
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, NewHashReader(bytes.NewReader(data), 5, md5Hex, "")); err != nil {
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
parts := []completePart{{PartNumber: 1, ETag: md5Hex}}
|
||||
parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
|
||||
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
if _, err := fs.CompleteMultipartUpload(bucketName, objectName, uploadID, parts); err != nil {
|
||||
@@ -226,6 +226,72 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
|
||||
func TestCompleteMultipartUpload(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer os.RemoveAll(disk)
|
||||
obj := initFSObjects(disk, t)
|
||||
|
||||
fs := obj.(*fsObjects)
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
data := []byte("12345")
|
||||
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Cannot create bucket, err: ", err)
|
||||
}
|
||||
|
||||
uploadID, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
md5Hex := getMD5Hash(data)
|
||||
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
|
||||
|
||||
if _, err := fs.CompleteMultipartUpload(bucketName, objectName, uploadID, parts); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
|
||||
func TestAbortMultipartUpload(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer os.RemoveAll(disk)
|
||||
obj := initFSObjects(disk, t)
|
||||
|
||||
fs := obj.(*fsObjects)
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
data := []byte("12345")
|
||||
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Cannot create bucket, err: ", err)
|
||||
}
|
||||
|
||||
uploadID, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
md5Hex := getMD5Hash(data)
|
||||
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
if err := fs.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestListMultipartUploadsFaultyDisk - test ListMultipartUploads with faulty disks
|
||||
func TestListMultipartUploadsFaultyDisk(t *testing.T) {
|
||||
// Prepare for tests
|
||||
@@ -251,7 +317,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
|
||||
md5Hex := getMD5Hash(data)
|
||||
sha256sum := ""
|
||||
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, NewHashReader(bytes.NewReader(data), 5, md5Hex, sha256sum)); err != nil {
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, sha256sum)); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
|
||||
+94
-32
@@ -27,6 +27,7 @@ import (
|
||||
"sort"
|
||||
"syscall"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
@@ -86,7 +87,7 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fi, err := osStat((fsPath))
|
||||
fi, err := os.Stat((fsPath))
|
||||
if err == nil {
|
||||
if !fi.IsDir() {
|
||||
return nil, syscall.ENOTDIR
|
||||
@@ -201,7 +202,7 @@ func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st, err := fsStatDir(bucketDir)
|
||||
st, err := fsStatVolume(bucketDir)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -230,7 +231,7 @@ func (fs fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
return bi, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
// As osStat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
|
||||
// As os.Stat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
|
||||
createdTime := st.ModTime()
|
||||
return BucketInfo{
|
||||
Name: bucket,
|
||||
@@ -255,7 +256,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
|
||||
continue
|
||||
}
|
||||
var fi os.FileInfo
|
||||
fi, err = fsStatDir(pathJoin(fs.fsPath, entry))
|
||||
fi, err = fsStatVolume(pathJoin(fs.fsPath, entry))
|
||||
// There seems like no practical reason to check for errors
|
||||
// at this point, if there are indeed errors we can simply
|
||||
// just ignore such buckets and list only those which
|
||||
@@ -266,7 +267,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
|
||||
}
|
||||
bucketInfos = append(bucketInfos, BucketInfo{
|
||||
Name: fi.Name(),
|
||||
// As osStat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
|
||||
// As os.Stat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
|
||||
Created: fi.ModTime(),
|
||||
})
|
||||
}
|
||||
@@ -361,7 +362,12 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
|
||||
pipeWriter.Close() // Close writer explicitly signalling we wrote all data.
|
||||
}()
|
||||
|
||||
objInfo, err := fs.PutObject(dstBucket, dstObject, NewHashReader(pipeReader, length, metadata["etag"], ""), metadata)
|
||||
hashReader, err := hash.NewReader(pipeReader, length, "", "")
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
|
||||
objInfo, err := fs.PutObject(dstBucket, dstObject, hashReader, metadata)
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
@@ -379,7 +385,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) {
|
||||
if err = checkGetObjArgs(bucket, object); err != nil {
|
||||
if err = checkBucketAndObjectNamesFS(bucket, object); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -397,6 +403,12 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
|
||||
return toObjectErr(traceError(errUnexpected), bucket, object)
|
||||
}
|
||||
|
||||
// If its a directory request, we return an empty body.
|
||||
if hasSuffix(object, slashSeparator) {
|
||||
_, err = writer.Write([]byte(""))
|
||||
return toObjectErr(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
if bucket != minioMetaBucket {
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
_, err = fs.rwPool.Open(fsMetaPath)
|
||||
@@ -440,6 +452,19 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
|
||||
// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
|
||||
func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
|
||||
fsMeta := fsMetaV1{}
|
||||
fi, err := fsStatDir(pathJoin(fs.fsPath, bucket, object))
|
||||
if err != nil && errorCause(err) != errFileAccessDenied {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
if fi != nil {
|
||||
// If file found and request was with object ending with "/", consider it
|
||||
// as directory and return object info
|
||||
if hasSuffix(object, slashSeparator) {
|
||||
return fsMeta.ToObjectInfo(bucket, object, fi), nil
|
||||
}
|
||||
return oi, toObjectErr(errFileNotFound, bucket, object)
|
||||
}
|
||||
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
|
||||
// Read `fs.json` to perhaps contend with
|
||||
@@ -464,7 +489,7 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
|
||||
}
|
||||
|
||||
// Stat the file to get file size.
|
||||
fi, err := fsStatFile(pathJoin(fs.fsPath, bucket, object))
|
||||
fi, err = fsStatFile(pathJoin(fs.fsPath, bucket, object))
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
@@ -472,9 +497,25 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
|
||||
return fsMeta.ToObjectInfo(bucket, object, fi), nil
|
||||
}
|
||||
|
||||
// Checks bucket and object name validity, returns nil if both are valid.
|
||||
func checkBucketAndObjectNamesFS(bucket, object string) error {
|
||||
// Verify if bucket is valid.
|
||||
if !IsValidBucketName(bucket) {
|
||||
return traceError(BucketNameInvalid{Bucket: bucket})
|
||||
}
|
||||
// Verify if object is valid.
|
||||
if len(object) == 0 {
|
||||
return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
|
||||
}
|
||||
if !IsValidObjectPrefix(object) {
|
||||
return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetObjectInfo - reads object metadata and replies back ObjectInfo.
|
||||
func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
|
||||
if err := checkGetObjArgs(bucket, object); err != nil {
|
||||
if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
|
||||
@@ -509,7 +550,12 @@ func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
|
||||
// until EOF, writes data directly to configured filesystem path.
|
||||
// Additionally writes `fs.json` which carries the necessary metadata
|
||||
// for future object operations.
|
||||
func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
|
||||
func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
|
||||
// No metadata is set, allocate a new one.
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string)
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// Validate if bucket name is valid and exists.
|
||||
@@ -517,6 +563,9 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
|
||||
return ObjectInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
fsMeta := newFSMetaV1()
|
||||
fsMeta.Meta = metadata
|
||||
|
||||
// This is a special case with size as '0' and object ends
|
||||
// with a slash separator, we treat it like a valid operation
|
||||
// and return success.
|
||||
@@ -525,7 +574,14 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
|
||||
if fs.parentDirIsObject(bucket, path.Dir(object)) {
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
|
||||
}
|
||||
return dirObjectInfo(bucket, object, data.Size(), metadata), nil
|
||||
if err = fsMkdirAll(pathJoin(fs.fsPath, bucket, object)); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
var fi os.FileInfo
|
||||
if fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, object)); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
return fsMeta.ToObjectInfo(bucket, object, fi), nil
|
||||
}
|
||||
|
||||
if err = checkPutObjectArgs(bucket, object, fs); err != nil {
|
||||
@@ -537,14 +593,11 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
|
||||
}
|
||||
|
||||
// No metadata is set, allocate a new one.
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string)
|
||||
// Validate input data size and it can never be less than zero.
|
||||
if data.Size() < 0 {
|
||||
return ObjectInfo{}, traceError(errInvalidArgument)
|
||||
}
|
||||
|
||||
fsMeta := newFSMetaV1()
|
||||
fsMeta.Meta = metadata
|
||||
|
||||
var wlk *lock.LockedFile
|
||||
if bucket != minioMetaBucket {
|
||||
bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
|
||||
@@ -574,6 +627,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
|
||||
if size := data.Size(); size > 0 && bufSize > size {
|
||||
bufSize = size
|
||||
}
|
||||
|
||||
buf := make([]byte, int(bufSize))
|
||||
fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
|
||||
bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size())
|
||||
@@ -583,6 +637,8 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
metadata["etag"] = hex.EncodeToString(data.MD5Current())
|
||||
|
||||
// Should return IncompleteBody{} error when reader has fewer
|
||||
// bytes than specified in request header.
|
||||
if bytesWritten < data.Size() {
|
||||
@@ -595,12 +651,6 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
|
||||
// nothing to delete.
|
||||
defer fsRemoveFile(fsTmpObjPath)
|
||||
|
||||
if err = data.Verify(); err != nil { // verify MD5 and SHA256
|
||||
return ObjectInfo{}, traceError(err)
|
||||
}
|
||||
|
||||
metadata["etag"] = hex.EncodeToString(data.MD5())
|
||||
|
||||
// Entire object was written to the temp location, now it's safe to rename it to the actual location.
|
||||
fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
|
||||
if err = fsRenameFile(fsTmpObjPath, fsNSObjPath); err != nil {
|
||||
@@ -627,7 +677,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *HashReader, me
|
||||
// DeleteObject - deletes an object from a bucket, this operation is destructive
|
||||
// and there are no rollbacks supported.
|
||||
func (fs fsObjects) DeleteObject(bucket, object string) error {
|
||||
if err := checkDelObjArgs(bucket, object); err != nil {
|
||||
if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -774,18 +824,30 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
|
||||
|
||||
// Convert entry to ObjectInfo
|
||||
entryToObjectInfo := func(entry string) (objInfo ObjectInfo, err error) {
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
// Object name needs to be full path.
|
||||
objInfo.Name = entry
|
||||
objInfo.IsDir = true
|
||||
return
|
||||
}
|
||||
|
||||
// Protect reading `fs.json`.
|
||||
// Protect the entry from concurrent deletes, or renames.
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, entry)
|
||||
if err = objectLock.GetRLock(globalListingTimeout); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
var fi os.FileInfo
|
||||
fi, err = fsStatDir(pathJoin(fs.fsPath, bucket, entry))
|
||||
objectLock.RUnlock()
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
// Success.
|
||||
return ObjectInfo{
|
||||
// Object name needs to be full path.
|
||||
Name: entry,
|
||||
Bucket: bucket,
|
||||
Size: fi.Size(),
|
||||
ModTime: fi.ModTime(),
|
||||
IsDir: fi.IsDir(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
var etag string
|
||||
etag, err = fs.getObjectETag(bucket, entry)
|
||||
objectLock.RUnlock()
|
||||
|
||||
+8
-8
@@ -46,7 +46,7 @@ func TestFSParentDirIsObject(t *testing.T) {
|
||||
}
|
||||
objectContent := "12345"
|
||||
objInfo, err := obj.PutObject(bucketName, objectName,
|
||||
NewHashReader(bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
|
||||
mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -135,7 +135,7 @@ func TestFSShutdown(t *testing.T) {
|
||||
fs := obj.(*fsObjects)
|
||||
objectContent := "12345"
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
obj.PutObject(bucketName, objectName, NewHashReader(bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
|
||||
obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil)
|
||||
return fs, disk
|
||||
}
|
||||
|
||||
@@ -205,7 +205,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
}
|
||||
|
||||
// With a regular object.
|
||||
_, err := obj.PutObject(bucketName+"non-existent", objectName, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
_, err := obj.PutObject(bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
@@ -214,7 +214,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
}
|
||||
|
||||
// With a directory object.
|
||||
_, err = obj.PutObject(bucketName+"non-existent", objectName+"/", NewHashReader(bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
|
||||
_, err = obj.PutObject(bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
@@ -222,11 +222,11 @@ func TestFSPutObject(t *testing.T) {
|
||||
t.Fatalf("Expected error type BucketNotFound, got %#v", err)
|
||||
}
|
||||
|
||||
_, err = obj.PutObject(bucketName, objectName, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
_, err = obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = obj.PutObject(bucketName, objectName+"/1", NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
_, err = obj.PutObject(bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backend corruption occurred")
|
||||
}
|
||||
@@ -241,7 +241,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
_, err = obj.PutObject(bucketName, objectName+"/1/", NewHashReader(bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
|
||||
_, err = obj.PutObject(bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backned corruption occurred")
|
||||
}
|
||||
@@ -269,7 +269,7 @@ func TestFSDeleteObject(t *testing.T) {
|
||||
objectName := "object"
|
||||
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
obj.PutObject(bucketName, objectName, NewHashReader(bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
|
||||
// Test with invalid bucket name
|
||||
if err := fs.DeleteObject("fo", objectName); !isSameType(errorCause(err), BucketNameInvalid{}) {
|
||||
|
||||
@@ -142,13 +142,6 @@ func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo,
|
||||
return bucketInfo, nil
|
||||
}
|
||||
|
||||
// AnonPutObject - SendPUT request without authentication.
|
||||
// This is needed when clients send PUT requests on objects that can be uploaded without auth.
|
||||
func (a *azureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
|
||||
// azure doesn't support anonymous put
|
||||
return ObjectInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetObject - SendGET request without authentication.
|
||||
// This is needed when clients send GET requests on objects that can be downloaded without auth.
|
||||
func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
|
||||
|
||||
@@ -1,43 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (a *azureObjects) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (a *azureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return nil, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (a *azureObjects) HealObject(bucket, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (a *azureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (a *azureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
|
||||
delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
return lmi, traceError(NotImplemented{})
|
||||
}
|
||||
+210
-70
@@ -27,18 +27,93 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
const globalAzureAPIVersion = "2016-05-31"
|
||||
const azureBlockSize = 100 * humanize.MiByte
|
||||
const metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
||||
const (
|
||||
globalAzureAPIVersion = "2016-05-31"
|
||||
azureBlockSize = 100 * humanize.MiByte
|
||||
metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
||||
azureBackend = "azure"
|
||||
)
|
||||
|
||||
func init() {
|
||||
const azureGatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENDPOINT:
|
||||
Azure server endpoint. Default ENDPOINT is https://core.windows.net
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of Azure storage.
|
||||
MINIO_SECRET_KEY: Password or secret key of Azure storage.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}} https://azure.example.com
|
||||
|
||||
`
|
||||
|
||||
MustRegisterGatewayCommand(cli.Command{
|
||||
Name: azureBackend,
|
||||
Usage: "Microsoft Azure Blob Storage.",
|
||||
Action: azureGatewayMain,
|
||||
CustomHelpTemplate: azureGatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway azure' command line.
|
||||
func azureGatewayMain(ctx *cli.Context) {
|
||||
// Validate gateway arguments.
|
||||
host := ctx.Args().First()
|
||||
// Validate gateway arguments.
|
||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||
|
||||
startGateway(ctx, &AzureGateway{host})
|
||||
}
|
||||
|
||||
// AzureGateway implements Gateway.
|
||||
type AzureGateway struct {
|
||||
host string
|
||||
}
|
||||
|
||||
// Name implements Gateway interface.
|
||||
func (g *AzureGateway) Name() string {
|
||||
return azureBackend
|
||||
}
|
||||
|
||||
// NewGatewayLayer initializes azure blob storage client and returns AzureObjects.
|
||||
func (g *AzureGateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||
return newAzureLayer(g.host)
|
||||
}
|
||||
|
||||
// s3MetaToAzureProperties converts metadata meant for S3 PUT/COPY
|
||||
// object into Azure data structures - BlobMetadata and
|
||||
@@ -55,31 +130,37 @@ const metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure
|
||||
//
|
||||
// Header names are canonicalized as in http.Header.
|
||||
func s3MetaToAzureProperties(s3Metadata map[string]string) (storage.BlobMetadata,
|
||||
storage.BlobProperties) {
|
||||
|
||||
// Azure does not permit user-defined metadata key names to
|
||||
// contain hyphens. So we map hyphens to underscores for
|
||||
// encryption headers. More such headers may be added in the
|
||||
// future.
|
||||
gatewayHeaders := map[string]string{
|
||||
"X-Amz-Meta-X-Amz-Key": "X-Amz-Meta-x_minio_key",
|
||||
"X-Amz-Meta-X-Amz-Matdesc": "X-Amz-Meta-x_minio_matdesc",
|
||||
"X-Amz-Meta-X-Amz-Iv": "X-Amz-Meta-x_minio_iv",
|
||||
storage.BlobProperties, error) {
|
||||
for k := range s3Metadata {
|
||||
if strings.Contains(k, "--") {
|
||||
return storage.BlobMetadata{}, storage.BlobProperties{}, traceError(UnsupportedMetadata{})
|
||||
}
|
||||
}
|
||||
|
||||
// Encoding technique for each key is used here is as follows
|
||||
// Each '-' is converted to '_'
|
||||
// Each '_' is converted to '__'
|
||||
// With this basic assumption here are some of the expected
|
||||
// translations for these keys.
|
||||
// i: 'x-S3cmd_attrs' -> o: 'x_s3cmd__attrs' (mixed)
|
||||
// i: 'x__test__value' -> o: 'x____test____value' (double '_')
|
||||
encodeKey := func(key string) string {
|
||||
tokens := strings.Split(key, "_")
|
||||
for i := range tokens {
|
||||
tokens[i] = strings.Replace(tokens[i], "-", "_", -1)
|
||||
}
|
||||
return strings.Join(tokens, "__")
|
||||
}
|
||||
var blobMeta storage.BlobMetadata = make(map[string]string)
|
||||
var props storage.BlobProperties
|
||||
for k, v := range s3Metadata {
|
||||
k = http.CanonicalHeaderKey(k)
|
||||
if nk, ok := gatewayHeaders[k]; ok {
|
||||
k = nk
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(k, "X-Amz-Meta-"):
|
||||
// Strip header prefix, to let Azure SDK
|
||||
// handle it for storage.
|
||||
k = strings.Replace(k, "X-Amz-Meta-", "", 1)
|
||||
blobMeta[k] = v
|
||||
blobMeta[encodeKey(k)] = v
|
||||
|
||||
// All cases below, extract common metadata that is
|
||||
// accepted by S3 into BlobProperties for setting on
|
||||
@@ -94,13 +175,13 @@ func s3MetaToAzureProperties(s3Metadata map[string]string) (storage.BlobMetadata
|
||||
case k == "Content-Length":
|
||||
// assume this doesn't fail
|
||||
props.ContentLength, _ = strconv.ParseInt(v, 10, 64)
|
||||
case k == "Content-MD5":
|
||||
case k == "Content-Md5":
|
||||
props.ContentMD5 = v
|
||||
case k == "Content-Type":
|
||||
props.ContentType = v
|
||||
}
|
||||
}
|
||||
return blobMeta, props
|
||||
return blobMeta, props, nil
|
||||
}
|
||||
|
||||
// azurePropertiesToS3Meta converts Azure metadata/properties to S3
|
||||
@@ -108,22 +189,26 @@ func s3MetaToAzureProperties(s3Metadata map[string]string) (storage.BlobMetadata
|
||||
// `.GetMetadata()` lower-cases all header keys, so this is taken into
|
||||
// account by this function.
|
||||
func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProperties) map[string]string {
|
||||
// Remap underscores to hyphens to restore encryption
|
||||
// headers. See s3MetaToAzureProperties for details.
|
||||
gatewayHeaders := map[string]string{
|
||||
"X-Amz-Meta-x_minio_key": "X-Amz-Meta-X-Amz-Key",
|
||||
"X-Amz-Meta-x_minio_matdesc": "X-Amz-Meta-X-Amz-Matdesc",
|
||||
"X-Amz-Meta-x_minio_iv": "X-Amz-Meta-X-Amz-Iv",
|
||||
// Decoding technique for each key is used here is as follows
|
||||
// Each '_' is converted to '-'
|
||||
// Each '__' is converted to '_'
|
||||
// With this basic assumption here are some of the expected
|
||||
// translations for these keys.
|
||||
// i: 'x_s3cmd__attrs' -> o: 'x-s3cmd_attrs' (mixed)
|
||||
// i: 'x____test____value' -> o: 'x__test__value' (double '_')
|
||||
decodeKey := func(key string) string {
|
||||
tokens := strings.Split(key, "__")
|
||||
for i := range tokens {
|
||||
tokens[i] = strings.Replace(tokens[i], "_", "-", -1)
|
||||
}
|
||||
return strings.Join(tokens, "_")
|
||||
}
|
||||
|
||||
s3Metadata := make(map[string]string)
|
||||
for k, v := range meta {
|
||||
// k's `x-ms-meta-` prefix is already stripped by
|
||||
// Azure SDK, so we add the AMZ prefix.
|
||||
k = "X-Amz-Meta-" + k
|
||||
if nk, ok := gatewayHeaders[k]; ok {
|
||||
k = nk
|
||||
}
|
||||
k = "X-Amz-Meta-" + decodeKey(k)
|
||||
k = http.CanonicalHeaderKey(k)
|
||||
s3Metadata[k] = v
|
||||
}
|
||||
@@ -151,13 +236,9 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper
|
||||
return s3Metadata
|
||||
}
|
||||
|
||||
// Append "-1" to etag so that clients do not interpret it as MD5.
|
||||
func azureToS3ETag(etag string) string {
|
||||
return canonicalizeETag(etag) + "-1"
|
||||
}
|
||||
|
||||
// azureObjects - Implements Object layer for Azure blob storage.
|
||||
type azureObjects struct {
|
||||
gatewayUnsupported
|
||||
client storage.BlobStorageClient // Azure sdk client
|
||||
}
|
||||
|
||||
@@ -408,7 +489,7 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max
|
||||
Name: object.Name,
|
||||
ModTime: time.Time(object.Properties.LastModified),
|
||||
Size: object.Properties.ContentLength,
|
||||
ETag: azureToS3ETag(object.Properties.Etag),
|
||||
ETag: toS3ETag(object.Properties.Etag),
|
||||
ContentType: object.Properties.ContentType,
|
||||
ContentEncoding: object.Properties.ContentEncoding,
|
||||
})
|
||||
@@ -462,8 +543,13 @@ func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimite
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
// startOffset cannot be negative.
|
||||
if startOffset < 0 {
|
||||
return toObjectErr(traceError(errUnexpected), bucket, object)
|
||||
}
|
||||
|
||||
blobRange := &storage.BlobRange{Start: uint64(startOffset)}
|
||||
if length > 0 && startOffset > 0 {
|
||||
if length > 0 {
|
||||
blobRange.End = uint64(startOffset + length - 1)
|
||||
}
|
||||
|
||||
@@ -498,7 +584,7 @@ func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo,
|
||||
objInfo = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
UserDefined: meta,
|
||||
ETag: azureToS3ETag(blob.Properties.Etag),
|
||||
ETag: toS3ETag(blob.Properties.Etag),
|
||||
ModTime: time.Time(blob.Properties.LastModified),
|
||||
Name: object,
|
||||
Size: blob.Properties.ContentLength,
|
||||
@@ -510,20 +596,16 @@ func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo,
|
||||
|
||||
// PutObject - Create a new blob with the incoming data,
|
||||
// uses Azure equivalent CreateBlockBlobFromReader.
|
||||
func (a *azureObjects) PutObject(bucket, object string, data *HashReader, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
delete(metadata, "etag")
|
||||
func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
blob.Metadata, blob.Properties = s3MetaToAzureProperties(metadata)
|
||||
blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(metadata)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(err, bucket, object)
|
||||
}
|
||||
err = blob.CreateBlockBlobFromReader(data, nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
if err = data.Verify(); err != nil {
|
||||
errorIf(err, "Verification of uploaded object data failed against client provided checksums.")
|
||||
derr := blob.Delete(nil)
|
||||
errorIf(derr, "Failed to delete blob when cleaning up a bad blob upload.")
|
||||
return ObjectInfo{}, azureToObjectError(traceError(err))
|
||||
}
|
||||
return a.GetObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
@@ -532,7 +614,10 @@ func (a *azureObjects) PutObject(bucket, object string, data *HashReader, metada
|
||||
func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
|
||||
destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
|
||||
azureMeta, props := s3MetaToAzureProperties(metadata)
|
||||
azureMeta, props, err := s3MetaToAzureProperties(metadata)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(err, srcBucket, srcObject)
|
||||
}
|
||||
destBlob.Metadata = azureMeta
|
||||
err = destBlob.Copy(srcBlobURL, nil)
|
||||
if err != nil {
|
||||
@@ -586,10 +671,6 @@ func (a *azureObjects) checkUploadIDExists(bucketName, objectName, uploadID stri
|
||||
|
||||
// NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
|
||||
func (a *azureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string)
|
||||
}
|
||||
|
||||
uploadID = mustGetAzureUploadID()
|
||||
if err = a.checkUploadIDExists(bucket, object, uploadID); err == nil {
|
||||
return "", traceError(errors.New("Upload ID name collision"))
|
||||
@@ -610,13 +691,8 @@ func (a *azureObjects) NewMultipartUpload(bucket, object string, metadata map[st
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// CopyObjectPart - Not implemented.
|
||||
func (a *azureObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return info, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// PutObjectPart - Use Azure equivalent PutBlockWithLength.
|
||||
func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *HashReader) (info PartInfo, err error) {
|
||||
func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
|
||||
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
|
||||
return info, err
|
||||
}
|
||||
@@ -625,10 +701,9 @@ func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int
|
||||
return info, err
|
||||
}
|
||||
|
||||
etag := data.md5Sum
|
||||
etag := data.MD5HexString()
|
||||
if etag == "" {
|
||||
// Generate random ETag.
|
||||
etag = getMD5Hash([]byte(mustGetUUID()))
|
||||
etag = genETag()
|
||||
}
|
||||
|
||||
subPartSize, subPartNumber := int64(azureBlockSize), 1
|
||||
@@ -650,13 +725,6 @@ func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int
|
||||
}
|
||||
subPartNumber++
|
||||
}
|
||||
if err = data.Verify(); err != nil {
|
||||
errorIf(err, "Verification of uploaded object data failed against client provided checksums.")
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
derr := blob.Delete(nil)
|
||||
errorIf(derr, "Failed to delete blob when cleaning up a bad blob upload.")
|
||||
return info, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
info.PartNumber = partID
|
||||
info.ETag = etag
|
||||
@@ -671,7 +739,76 @@ func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumb
|
||||
return result, err
|
||||
}
|
||||
|
||||
// It's decided not to support List Object Parts, hence returning empty result.
|
||||
result.Bucket = bucket
|
||||
result.Object = object
|
||||
result.UploadID = uploadID
|
||||
result.MaxParts = maxParts
|
||||
|
||||
objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
// Build a sorted list of parts and return the requested entries.
|
||||
partsMap := make(map[int]PartInfo)
|
||||
for _, block := range resp.UncommittedBlocks {
|
||||
var partNumber int
|
||||
var parsedUploadID string
|
||||
var md5Hex string
|
||||
if partNumber, _, parsedUploadID, md5Hex, err = azureParseBlockID(block.Name); err != nil {
|
||||
return result, azureToObjectError(traceError(errUnexpected), bucket, object)
|
||||
}
|
||||
if parsedUploadID != uploadID {
|
||||
continue
|
||||
}
|
||||
part, ok := partsMap[partNumber]
|
||||
if !ok {
|
||||
partsMap[partNumber] = PartInfo{
|
||||
PartNumber: partNumber,
|
||||
Size: block.Size,
|
||||
ETag: md5Hex,
|
||||
}
|
||||
continue
|
||||
}
|
||||
if part.ETag != md5Hex {
|
||||
// If two parts of same partNumber were uploaded with different contents
|
||||
// return error as we won't be able to decide which the latest part is.
|
||||
return result, azureToObjectError(traceError(errUnexpected), bucket, object)
|
||||
}
|
||||
part.Size += block.Size
|
||||
partsMap[partNumber] = part
|
||||
}
|
||||
var parts []PartInfo
|
||||
for _, part := range partsMap {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
sort.SliceStable(parts, func(i int, j int) bool {
|
||||
return parts[i].PartNumber < parts[j].PartNumber
|
||||
})
|
||||
partsCount := 0
|
||||
i := 0
|
||||
if partNumberMarker != 0 {
|
||||
// If the marker was set, skip the entries till the marker.
|
||||
for _, part := range parts {
|
||||
i++
|
||||
if part.PartNumber == partNumberMarker {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
for partsCount < maxParts && i < len(parts) {
|
||||
result.Parts = append(result.Parts, parts[i])
|
||||
i++
|
||||
partsCount++
|
||||
}
|
||||
|
||||
if i < len(parts) {
|
||||
result.IsTruncated = true
|
||||
if partsCount != 0 {
|
||||
result.NextPartNumberMarker = result.Parts[partsCount-1].PartNumber
|
||||
}
|
||||
}
|
||||
result.PartNumberMarker = partNumberMarker
|
||||
return result, nil
|
||||
}
|
||||
|
||||
@@ -689,7 +826,7 @@ func (a *azureObjects) AbortMultipartUpload(bucket, object, uploadID string) (er
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - Use Azure equivalent PutBlockList.
|
||||
func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
|
||||
metadataObject := getAzureMetadataObjectName(object, uploadID)
|
||||
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
|
||||
return objInfo, err
|
||||
@@ -782,7 +919,10 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
if len(metadata.Metadata) > 0 {
|
||||
objBlob.Metadata, objBlob.Properties = s3MetaToAzureProperties(metadata.Metadata)
|
||||
objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(metadata.Metadata)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(err, bucket, object)
|
||||
}
|
||||
err = objBlob.SetProperties(nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
|
||||
+46
-26
@@ -25,29 +25,15 @@ import (
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
)
|
||||
|
||||
// Test azureToS3ETag.
|
||||
func TestAzureToS3ETag(t *testing.T) {
|
||||
tests := []struct {
|
||||
etag string
|
||||
expected string
|
||||
}{
|
||||
{`"etag"`, `etag-1`},
|
||||
{"etag", "etag-1"},
|
||||
}
|
||||
for i, test := range tests {
|
||||
got := azureToS3ETag(test.etag)
|
||||
if got != test.expected {
|
||||
t.Errorf("test %d: got:%s expected:%s", i+1, got, test.expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test canonical metadata.
|
||||
func TestS3MetaToAzureProperties(t *testing.T) {
|
||||
headers := map[string]string{
|
||||
"accept-encoding": "gzip",
|
||||
"content-encoding": "gzip",
|
||||
"X-Amz-Meta-Hdr": "value",
|
||||
"X-Amz-Meta-X_test_key": "value",
|
||||
"X-Amz-Meta-X__test__key": "value",
|
||||
"X-Amz-Meta-X-Test__key": "value",
|
||||
"X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"X-Amz-Meta-X-Amz-Matdesc": "{}",
|
||||
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
@@ -55,28 +41,62 @@ func TestS3MetaToAzureProperties(t *testing.T) {
|
||||
// Only X-Amz-Meta- prefixed entries will be returned in
|
||||
// Metadata (without the prefix!)
|
||||
expectedHeaders := map[string]string{
|
||||
"Hdr": "value",
|
||||
"x_minio_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"x_minio_matdesc": "{}",
|
||||
"x_minio_iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
"Hdr": "value",
|
||||
"X__test__key": "value",
|
||||
"X____test____key": "value",
|
||||
"X_Test____key": "value",
|
||||
"X_Amz_Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"X_Amz_Matdesc": "{}",
|
||||
"X_Amz_Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
}
|
||||
meta, _, err := s3MetaToAzureProperties(headers)
|
||||
if err != nil {
|
||||
t.Fatalf("Test failed, with %s", err)
|
||||
}
|
||||
meta, _ := s3MetaToAzureProperties(headers)
|
||||
if !reflect.DeepEqual(map[string]string(meta), expectedHeaders) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, meta)
|
||||
}
|
||||
headers = map[string]string{
|
||||
"invalid--meta": "value",
|
||||
}
|
||||
_, _, err = s3MetaToAzureProperties(headers)
|
||||
if err = errorCause(err); err != nil {
|
||||
if _, ok := err.(UnsupportedMetadata); !ok {
|
||||
t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
|
||||
}
|
||||
}
|
||||
|
||||
headers = map[string]string{
|
||||
"content-md5": "Dce7bmCX61zvxzP5QmfelQ==",
|
||||
}
|
||||
_, props, err := s3MetaToAzureProperties(headers)
|
||||
if err != nil {
|
||||
t.Fatalf("Test failed, with %s", err)
|
||||
}
|
||||
if props.ContentMD5 != headers["content-md5"] {
|
||||
t.Fatalf("Test failed, expected %s, got %s", headers["content-md5"], props.ContentMD5)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzurePropertiesToS3Meta(t *testing.T) {
|
||||
// Just one testcase. Adding more test cases does not add value to the testcase
|
||||
// as azureToS3Metadata() just adds a prefix.
|
||||
metadata := map[string]string{
|
||||
"First-Name": "myname",
|
||||
"x_minio_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"x_minio_matdesc": "{}",
|
||||
"x_minio_iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
"first_name": "myname",
|
||||
"x_test_key": "value",
|
||||
"x_test__key": "value",
|
||||
"x__test__key": "value",
|
||||
"x____test____key": "value",
|
||||
"x_amz_key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"x_amz_matdesc": "{}",
|
||||
"x_amz_iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
}
|
||||
expectedMeta := map[string]string{
|
||||
"X-Amz-Meta-First-Name": "myname",
|
||||
"X-Amz-Meta-X-Test-Key": "value",
|
||||
"X-Amz-Meta-X-Test_key": "value",
|
||||
"X-Amz-Meta-X_test_key": "value",
|
||||
"X-Amz-Meta-X__test__key": "value",
|
||||
"X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"X-Amz-Meta-X-Amz-Matdesc": "{}",
|
||||
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// mkRange converts offset, size into Range header equivalent.
|
||||
func mkRange(offset, size int64) string {
|
||||
if offset == 0 && size == 0 {
|
||||
return ""
|
||||
}
|
||||
if size == 0 {
|
||||
return fmt.Sprintf("%s%d-", byteRangePrefix, offset)
|
||||
}
|
||||
return fmt.Sprintf("%s%d-%d", byteRangePrefix, offset, offset+size-1)
|
||||
}
|
||||
|
||||
// AnonGetObject - performs a plain http GET request on a public resource,
|
||||
// fails if the resource is not public.
|
||||
func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
|
||||
req, err := http.NewRequest("GET", uri, nil)
|
||||
if err != nil {
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
rng := mkRange(startOffset, length)
|
||||
if rng != "" {
|
||||
req.Header.Set("Range", rng)
|
||||
}
|
||||
resp, err := l.anonClient.Do(req)
|
||||
if err != nil {
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
|
||||
}
|
||||
_, err = io.Copy(writer, resp.Body)
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// Converts http Header into ObjectInfo. This function looks for all the
|
||||
// standard Backblaze B2 headers to convert into ObjectInfo.
|
||||
//
|
||||
// Content-Length is converted to Size.
|
||||
// X-Bz-Upload-Timestamp is converted to ModTime.
|
||||
// X-Bz-Info-<header>:<value> is converted to <header>:<value>
|
||||
// Content-Type is converted to ContentType.
|
||||
// X-Bz-Content-Sha1 is converted to ETag.
|
||||
func headerToObjectInfo(bucket, object string, header http.Header) (objInfo ObjectInfo, err error) {
|
||||
clen, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// Converting upload timestamp in milliseconds to a time.Time value for ObjectInfo.ModTime.
|
||||
timeStamp, err := strconv.ParseInt(header.Get("X-Bz-Upload-Timestamp"), 10, 64)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// Populate user metadata by looking for all the X-Bz-Info-<name>
|
||||
// HTTP headers, ignore other headers since they have their own
|
||||
// designated meaning, for more details refer B2 API documentation.
|
||||
userMetadata := make(map[string]string)
|
||||
for key := range header {
|
||||
if strings.HasPrefix(key, "X-Bz-Info-") {
|
||||
var name string
|
||||
name, err = url.QueryUnescape(strings.TrimPrefix(key, "X-Bz-Info-"))
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
var val string
|
||||
val, err = url.QueryUnescape(header.Get(key))
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
userMetadata[name] = val
|
||||
}
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
ContentType: header.Get("Content-Type"),
|
||||
ModTime: time.Unix(0, 0).Add(time.Duration(timeStamp) * time.Millisecond),
|
||||
Size: clen,
|
||||
ETag: header.Get("X-Bz-File-Id"),
|
||||
UserDefined: userMetadata,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AnonGetObjectInfo - performs a plain http HEAD request on a public resource,
|
||||
// fails if the resource is not public.
|
||||
func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
|
||||
uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
|
||||
req, err := http.NewRequest("HEAD", uri, nil)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
resp, err := l.anonClient.Do(req)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return objInfo, b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
|
||||
}
|
||||
return headerToObjectInfo(bucket, object, resp.Header)
|
||||
}
|
||||
@@ -0,0 +1,759 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha1"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
b2 "github.com/minio/blazer/base"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
h2 "github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// Supported bucket types by B2 backend.
|
||||
const (
|
||||
bucketTypePrivate = "allPrivate"
|
||||
bucketTypeReadOnly = "allPublic"
|
||||
b2Backend = "b2"
|
||||
)
|
||||
|
||||
func init() {
|
||||
const b2GatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}}
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: B2 account id.
|
||||
MINIO_SECRET_KEY: B2 application key.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for B2 backend.
|
||||
$ export MINIO_ACCESS_KEY=accountID
|
||||
$ export MINIO_SECRET_KEY=applicationKey
|
||||
$ {{.HelpName}}
|
||||
|
||||
`
|
||||
|
||||
MustRegisterGatewayCommand(cli.Command{
|
||||
Name: b2Backend,
|
||||
Usage: "Backblaze B2.",
|
||||
Action: b2GatewayMain,
|
||||
CustomHelpTemplate: b2GatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway b2' command line.
|
||||
func b2GatewayMain(ctx *cli.Context) {
|
||||
startGateway(ctx, &B2Gateway{})
|
||||
}
|
||||
|
||||
// B2Gateway implements Gateway.
|
||||
type B2Gateway struct{}
|
||||
|
||||
// Name implements Gateway interface.
|
||||
func (g *B2Gateway) Name() string {
|
||||
return b2Backend
|
||||
}
|
||||
|
||||
// NewGatewayLayer returns b2 gateway layer, implements GatewayLayer interface to
|
||||
// talk to B2 remote backend.
|
||||
func (g *B2Gateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
||||
return newB2GatewayLayer()
|
||||
}
|
||||
|
||||
// b2Object implements gateway for Minio and BackBlaze B2 compatible object storage servers.
|
||||
type b2Objects struct {
|
||||
gatewayUnsupported
|
||||
mu sync.Mutex
|
||||
creds auth.Credentials
|
||||
b2Client *b2.B2
|
||||
anonClient *http.Client
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// newB2GatewayLayer returns b2 gateway layer.
|
||||
func newB2GatewayLayer() (GatewayLayer, error) {
|
||||
ctx := context.Background()
|
||||
creds := serverConfig.GetCredential()
|
||||
|
||||
client, err := b2.AuthorizeAccount(ctx, creds.AccessKey, creds.SecretKey, b2.Transport(newCustomHTTPTransport()))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &b2Objects{
|
||||
creds: creds,
|
||||
b2Client: client,
|
||||
anonClient: &http.Client{
|
||||
Transport: newCustomHTTPTransport(),
|
||||
},
|
||||
ctx: ctx,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Convert B2 errors to minio object layer errors.
|
||||
func b2ToObjectError(err error, params ...string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
e, ok := err.(*Error)
|
||||
if !ok {
|
||||
// Code should be fixed if this function is called without doing traceError()
|
||||
// Else handling different situations in this function makes this function complicated.
|
||||
errorIf(err, "Expected type *Error")
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.e
|
||||
bucket := ""
|
||||
object := ""
|
||||
uploadID := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
if len(params) == 3 {
|
||||
uploadID = params[2]
|
||||
}
|
||||
|
||||
// Following code is a non-exhaustive check to convert
|
||||
// B2 errors into S3 compatible errors.
|
||||
//
|
||||
// For a more complete information - https://www.backblaze.com/b2/docs/
|
||||
statusCode, code, msg := b2.Code(err)
|
||||
if statusCode == 0 {
|
||||
// We don't interpret non B2 errors. B2 errors have statusCode
|
||||
// to help us convert them to S3 object errors.
|
||||
return e
|
||||
}
|
||||
|
||||
switch code {
|
||||
case "duplicate_bucket_name":
|
||||
err = BucketAlreadyOwnedByYou{Bucket: bucket}
|
||||
case "bad_request":
|
||||
if object != "" {
|
||||
err = ObjectNameInvalid{bucket, object}
|
||||
} else if bucket != "" {
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
}
|
||||
case "bad_bucket_id":
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
case "file_not_present", "not_found":
|
||||
err = ObjectNotFound{bucket, object}
|
||||
case "cannot_delete_non_empty_bucket":
|
||||
err = BucketNotEmpty{bucket, ""}
|
||||
}
|
||||
|
||||
// Special interpretation like this is required for Multipart sessions.
|
||||
if strings.Contains(msg, "No active upload for") && uploadID != "" {
|
||||
err = InvalidUploadID{uploadID}
|
||||
}
|
||||
|
||||
e.e = err
|
||||
return e
|
||||
}
|
||||
|
||||
// Shutdown saves any gateway metadata to disk
|
||||
// if necessary and reload upon next restart.
|
||||
func (l *b2Objects) Shutdown() error {
|
||||
// TODO
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageInfo is not relevant to B2 backend.
|
||||
func (l *b2Objects) StorageInfo() (si StorageInfo) {
|
||||
return si
|
||||
}
|
||||
|
||||
// MakeBucket creates a new container on B2 backend.
|
||||
func (l *b2Objects) MakeBucketWithLocation(bucket, location string) error {
|
||||
// location is ignored for B2 backend.
|
||||
|
||||
// All buckets are set to private by default.
|
||||
_, err := l.b2Client.CreateBucket(l.ctx, bucket, bucketTypePrivate, nil, nil)
|
||||
return b2ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
func (l *b2Objects) reAuthorizeAccount() error {
|
||||
client, err := b2.AuthorizeAccount(l.ctx, l.creds.AccessKey, l.creds.SecretKey, b2.Transport(newCustomHTTPTransport()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
l.mu.Lock()
|
||||
l.b2Client.Update(client)
|
||||
l.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// listBuckets is a wrapper similar to ListBuckets, which re-authorizes
|
||||
// the account and updates the B2 client safely. Once successfully
|
||||
// authorized performs the call again and returns list of buckets.
|
||||
// For any errors which are not actionable we return an error.
|
||||
func (l *b2Objects) listBuckets(err error) ([]*b2.Bucket, error) {
|
||||
if err != nil {
|
||||
if b2.Action(err) != b2.ReAuthenticate {
|
||||
return nil, err
|
||||
}
|
||||
if rerr := l.reAuthorizeAccount(); rerr != nil {
|
||||
return nil, rerr
|
||||
}
|
||||
}
|
||||
bktList, lerr := l.b2Client.ListBuckets(l.ctx)
|
||||
if lerr != nil {
|
||||
return l.listBuckets(lerr)
|
||||
}
|
||||
return bktList, nil
|
||||
}
|
||||
|
||||
// Bucket - is a helper which provides a *Bucket instance
|
||||
// for performing an API operation. B2 API doesn't
|
||||
// provide a direct way to access the bucket so we need
|
||||
// to employ following technique.
|
||||
func (l *b2Objects) Bucket(bucket string) (*b2.Bucket, error) {
|
||||
bktList, err := l.listBuckets(nil)
|
||||
if err != nil {
|
||||
return nil, b2ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
for _, bkt := range bktList {
|
||||
if bkt.Name == bucket {
|
||||
return bkt, nil
|
||||
}
|
||||
}
|
||||
return nil, traceError(BucketNotFound{Bucket: bucket})
|
||||
}
|
||||
|
||||
// GetBucketInfo gets bucket metadata..
|
||||
func (l *b2Objects) GetBucketInfo(bucket string) (bi BucketInfo, err error) {
|
||||
if _, err = l.Bucket(bucket); err != nil {
|
||||
return bi, err
|
||||
}
|
||||
return BucketInfo{
|
||||
Name: bucket,
|
||||
Created: time.Unix(0, 0),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ListBuckets lists all B2 buckets
|
||||
func (l *b2Objects) ListBuckets() ([]BucketInfo, error) {
|
||||
bktList, err := l.listBuckets(nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var bktInfo []BucketInfo
|
||||
for _, bkt := range bktList {
|
||||
bktInfo = append(bktInfo, BucketInfo{
|
||||
Name: bkt.Name,
|
||||
Created: time.Unix(0, 0),
|
||||
})
|
||||
}
|
||||
return bktInfo, nil
|
||||
}
|
||||
|
||||
// DeleteBucket deletes a bucket on B2
|
||||
func (l *b2Objects) DeleteBucket(bucket string) error {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = bkt.DeleteBucket(l.ctx)
|
||||
return b2ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
// ListObjects lists all objects in B2 bucket filtered by prefix, returns upto at max 1000 entries at a time.
|
||||
func (l *b2Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return loi, err
|
||||
}
|
||||
loi = ListObjectsInfo{}
|
||||
files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, marker, prefix, delimiter)
|
||||
if lerr != nil {
|
||||
return loi, b2ToObjectError(traceError(lerr), bucket)
|
||||
}
|
||||
loi.IsTruncated = next != ""
|
||||
loi.NextMarker = next
|
||||
for _, file := range files {
|
||||
switch file.Status {
|
||||
case "folder":
|
||||
loi.Prefixes = append(loi.Prefixes, file.Name)
|
||||
case "upload":
|
||||
loi.Objects = append(loi.Objects, ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: file.Name,
|
||||
ModTime: file.Timestamp,
|
||||
Size: file.Size,
|
||||
ETag: toS3ETag(file.Info.ID),
|
||||
ContentType: file.Info.ContentType,
|
||||
UserDefined: file.Info.Info,
|
||||
})
|
||||
}
|
||||
}
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
// ListObjectsV2 lists all objects in B2 bucket filtered by prefix, returns upto max 1000 entries at a time.
|
||||
func (l *b2Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
|
||||
fetchOwner bool, startAfter string) (loi ListObjectsV2Info, err error) {
|
||||
// fetchOwner, startAfter are not supported and unused.
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return loi, err
|
||||
}
|
||||
loi = ListObjectsV2Info{}
|
||||
files, next, lerr := bkt.ListFileNames(l.ctx, maxKeys, continuationToken, prefix, delimiter)
|
||||
if lerr != nil {
|
||||
return loi, b2ToObjectError(traceError(lerr), bucket)
|
||||
}
|
||||
loi.IsTruncated = next != ""
|
||||
loi.ContinuationToken = continuationToken
|
||||
loi.NextContinuationToken = next
|
||||
for _, file := range files {
|
||||
switch file.Status {
|
||||
case "folder":
|
||||
loi.Prefixes = append(loi.Prefixes, file.Name)
|
||||
case "upload":
|
||||
loi.Objects = append(loi.Objects, ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: file.Name,
|
||||
ModTime: file.Timestamp,
|
||||
Size: file.Size,
|
||||
ETag: toS3ETag(file.Info.ID),
|
||||
ContentType: file.Info.ContentType,
|
||||
UserDefined: file.Info.Info,
|
||||
})
|
||||
}
|
||||
}
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
// GetObject reads an object from B2. Supports additional
|
||||
// parameters like offset and length which are synonymous with
|
||||
// HTTP Range requests.
|
||||
//
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (l *b2Objects) GetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader, err := bkt.DownloadFileByName(l.ctx, object, startOffset, length)
|
||||
if err != nil {
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
defer reader.Close()
|
||||
_, err = io.Copy(writer, reader)
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// GetObjectInfo reads object info and replies back ObjectInfo
|
||||
func (l *b2Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
f, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
f.Close()
|
||||
fi, err := bkt.File(f.ID, object).GetFileInfo(l.ctx)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
objInfo = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
ETag: toS3ETag(fi.ID),
|
||||
Size: fi.Size,
|
||||
ModTime: fi.Timestamp,
|
||||
ContentType: fi.ContentType,
|
||||
UserDefined: fi.Info,
|
||||
}
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// In B2 - You must always include the X-Bz-Content-Sha1 header with
|
||||
// your upload request. The value you provide can be:
|
||||
// (1) the 40-character hex checksum of the file,
|
||||
// (2) the string hex_digits_at_end, or
|
||||
// (3) the string do_not_verify.
|
||||
// For more reference - https://www.backblaze.com/b2/docs/uploading.html
|
||||
//
|
||||
const (
|
||||
sha1NoVerify = "do_not_verify"
|
||||
sha1AtEOF = "hex_digits_at_end"
|
||||
)
|
||||
|
||||
// With the second option mentioned above, you append the 40-character hex sha1
|
||||
// to the end of the request body, immediately after the contents of the file
|
||||
// being uploaded. Note that the content length is the size of the file plus 40
|
||||
// of the original size of the reader.
|
||||
//
|
||||
// newB2Reader implements a B2 compatible reader by wrapping the hash.Reader into
|
||||
// a new io.Reader which will emit out the sha1 hex digits at io.EOF.
|
||||
// It also means that your overall content size is now original size + 40 bytes.
|
||||
// Additionally this reader also verifies Hash encapsulated inside hash.Reader
|
||||
// at io.EOF if the verification failed we return an error and do not send
|
||||
// the content to server.
|
||||
func newB2Reader(r *h2.Reader, size int64) *B2Reader {
|
||||
return &B2Reader{
|
||||
r: r,
|
||||
size: size,
|
||||
sha1Hash: sha1.New(),
|
||||
}
|
||||
}
|
||||
|
||||
// B2Reader - is a Reader wraps the hash.Reader which will emit out the sha1
|
||||
// hex digits at io.EOF. It also means that your overall content size is
|
||||
// now original size + 40 bytes. Additionally this reader also verifies
|
||||
// Hash encapsulated inside hash.Reader at io.EOF if the verification
|
||||
// failed we return an error and do not send the content to server.
|
||||
type B2Reader struct {
|
||||
r *h2.Reader
|
||||
size int64
|
||||
sha1Hash hash.Hash
|
||||
|
||||
isEOF bool
|
||||
buf *strings.Reader
|
||||
}
|
||||
|
||||
// Size - Returns the total size of Reader.
|
||||
func (nb *B2Reader) Size() int64 { return nb.size + 40 }
|
||||
func (nb *B2Reader) Read(p []byte) (int, error) {
|
||||
if nb.isEOF {
|
||||
return nb.buf.Read(p)
|
||||
}
|
||||
// Read into hash to update the on going checksum.
|
||||
n, err := io.TeeReader(nb.r, nb.sha1Hash).Read(p)
|
||||
if err == io.EOF {
|
||||
// Stream is not corrupted on this end
|
||||
// now fill in the last 40 bytes of sha1 hex
|
||||
// so that the server can verify the stream on
|
||||
// their end.
|
||||
err = nil
|
||||
nb.isEOF = true
|
||||
nb.buf = strings.NewReader(fmt.Sprintf("%x", nb.sha1Hash.Sum(nil)))
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
|
||||
// PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
|
||||
func (l *b2Objects) PutObject(bucket string, object string, data *h2.Reader, metadata map[string]string) (ObjectInfo, error) {
|
||||
var objInfo ObjectInfo
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
contentType := metadata["content-type"]
|
||||
delete(metadata, "content-type")
|
||||
|
||||
var u *b2.URL
|
||||
u, err = bkt.GetUploadURL(l.ctx)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
hr := newB2Reader(data, data.Size())
|
||||
var f *b2.File
|
||||
f, err = u.UploadFile(l.ctx, hr, int(hr.Size()), object, contentType, sha1AtEOF, metadata)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
var fi *b2.FileInfo
|
||||
fi, err = f.GetFileInfo(l.ctx)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
ETag: toS3ETag(fi.ID),
|
||||
Size: fi.Size,
|
||||
ModTime: fi.Timestamp,
|
||||
ContentType: fi.ContentType,
|
||||
UserDefined: fi.Info,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CopyObject copies a blob from source container to destination container.
|
||||
func (l *b2Objects) CopyObject(srcBucket string, srcObject string, dstBucket string,
|
||||
dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
return objInfo, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// DeleteObject deletes a blob in bucket
|
||||
func (l *b2Objects) DeleteObject(bucket string, object string) error {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
reader, err := bkt.DownloadFileByName(l.ctx, object, 0, 1)
|
||||
if err != nil {
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
io.Copy(ioutil.Discard, reader)
|
||||
reader.Close()
|
||||
err = bkt.File(reader.ID, object).DeleteFileVersion(l.ctx)
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// ListMultipartUploads lists all multipart uploads.
|
||||
func (l *b2Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string,
|
||||
delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
|
||||
// keyMarker, prefix, delimiter are all ignored, Backblaze B2 doesn't support any
|
||||
// of these parameters only equivalent parameter is uploadIDMarker.
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return lmi, err
|
||||
}
|
||||
// The maximum number of files to return from this call.
|
||||
// The default value is 100, and the maximum allowed is 100.
|
||||
if maxUploads > 100 {
|
||||
maxUploads = 100
|
||||
}
|
||||
largeFiles, nextMarker, err := bkt.ListUnfinishedLargeFiles(l.ctx, uploadIDMarker, maxUploads)
|
||||
if err != nil {
|
||||
return lmi, b2ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
lmi = ListMultipartsInfo{
|
||||
MaxUploads: maxUploads,
|
||||
}
|
||||
if nextMarker != "" {
|
||||
lmi.IsTruncated = true
|
||||
lmi.NextUploadIDMarker = nextMarker
|
||||
}
|
||||
for _, largeFile := range largeFiles {
|
||||
lmi.Uploads = append(lmi.Uploads, MultipartInfo{
|
||||
Object: largeFile.Name,
|
||||
UploadID: largeFile.ID,
|
||||
Initiated: largeFile.Timestamp,
|
||||
})
|
||||
}
|
||||
return lmi, nil
|
||||
}
|
||||
|
||||
// NewMultipartUpload upload object in multiple parts, uses B2's LargeFile upload API.
|
||||
// Large files can range in size from 5MB to 10TB.
|
||||
// Each large file must consist of at least 2 parts, and all of the parts except the
|
||||
// last one must be at least 5MB in size. The last part must contain at least one byte.
|
||||
// For more information - https://www.backblaze.com/b2/docs/large_files.html
|
||||
func (l *b2Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (string, error) {
|
||||
var uploadID string
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return uploadID, err
|
||||
}
|
||||
|
||||
contentType := metadata["content-type"]
|
||||
delete(metadata, "content-type")
|
||||
lf, err := bkt.StartLargeFile(l.ctx, object, contentType, metadata)
|
||||
if err != nil {
|
||||
return uploadID, b2ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return lf.ID, nil
|
||||
}
|
||||
|
||||
// CopyObjectPart copy part of object to other bucket and object.
|
||||
func (l *b2Objects) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string,
|
||||
uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return PartInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
|
||||
func (l *b2Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *h2.Reader) (pi PartInfo, err error) {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return pi, err
|
||||
}
|
||||
|
||||
fc, err := bkt.File(uploadID, object).CompileParts(0, nil).GetUploadPartURL(l.ctx)
|
||||
if err != nil {
|
||||
return pi, b2ToObjectError(traceError(err), bucket, object, uploadID)
|
||||
}
|
||||
|
||||
hr := newB2Reader(data, data.Size())
|
||||
sha1, err := fc.UploadPart(l.ctx, hr, sha1AtEOF, int(hr.Size()), partID)
|
||||
if err != nil {
|
||||
return pi, b2ToObjectError(traceError(err), bucket, object, uploadID)
|
||||
}
|
||||
|
||||
return PartInfo{
|
||||
PartNumber: partID,
|
||||
LastModified: UTCNow(),
|
||||
ETag: toS3ETag(sha1),
|
||||
Size: data.Size(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
|
||||
func (l *b2Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return lpi, err
|
||||
}
|
||||
lpi = ListPartsInfo{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
UploadID: uploadID,
|
||||
MaxParts: maxParts,
|
||||
PartNumberMarker: partNumberMarker,
|
||||
}
|
||||
// startPartNumber must be in the range 1 - 10000 for B2.
|
||||
partNumberMarker++
|
||||
partsList, next, err := bkt.File(uploadID, object).ListParts(l.ctx, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
return lpi, b2ToObjectError(traceError(err), bucket, object, uploadID)
|
||||
}
|
||||
if next != 0 {
|
||||
lpi.IsTruncated = true
|
||||
lpi.NextPartNumberMarker = next
|
||||
}
|
||||
for _, part := range partsList {
|
||||
lpi.Parts = append(lpi.Parts, PartInfo{
|
||||
PartNumber: part.Number,
|
||||
ETag: toS3ETag(part.SHA1),
|
||||
Size: part.Size,
|
||||
})
|
||||
}
|
||||
return lpi, nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload aborts a on going multipart upload, uses B2's LargeFile upload API.
|
||||
func (l *b2Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = bkt.File(uploadID, object).CompileParts(0, nil).CancelLargeFile(l.ctx)
|
||||
return b2ToObjectError(traceError(err), bucket, object, uploadID)
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
|
||||
func (l *b2Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return oi, err
|
||||
}
|
||||
hashes := make(map[int]string)
|
||||
for i, uploadedPart := range uploadedParts {
|
||||
// B2 requires contigous part numbers starting with 1, they do not support
|
||||
// hand picking part numbers, we return an S3 compatible error instead.
|
||||
if i+1 != uploadedPart.PartNumber {
|
||||
return oi, b2ToObjectError(traceError(InvalidPart{}), bucket, object, uploadID)
|
||||
}
|
||||
|
||||
// Trim "-1" suffix in ETag as PutObjectPart() treats B2 returned SHA1 as ETag.
|
||||
hashes[uploadedPart.PartNumber] = strings.TrimSuffix(uploadedPart.ETag, "-1")
|
||||
}
|
||||
|
||||
if _, err = bkt.File(uploadID, object).CompileParts(0, hashes).FinishLargeFile(l.ctx); err != nil {
|
||||
return oi, b2ToObjectError(traceError(err), bucket, object, uploadID)
|
||||
}
|
||||
|
||||
return l.GetObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
// SetBucketPolicies - B2 supports 2 types of bucket policies:
|
||||
// bucketType.AllPublic - bucketTypeReadOnly means that anybody can download the files is the bucket;
|
||||
// bucketType.AllPrivate - bucketTypePrivate means that you need an authorization token to download them.
|
||||
// Default is AllPrivate for all buckets.
|
||||
func (l *b2Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
|
||||
var policies []BucketAccessPolicy
|
||||
|
||||
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
|
||||
policies = append(policies, BucketAccessPolicy{
|
||||
Prefix: prefix,
|
||||
Policy: policy,
|
||||
})
|
||||
}
|
||||
prefix := bucket + "/*" // For all objects inside the bucket.
|
||||
if len(policies) != 1 {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
if policies[0].Prefix != prefix {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
if policies[0].Policy != policy.BucketPolicyReadOnly {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bkt.Type = bucketTypeReadOnly
|
||||
_, err = bkt.Update(l.ctx)
|
||||
return b2ToObjectError(traceError(err))
|
||||
}
|
||||
|
||||
// GetBucketPolicies, returns the current bucketType from B2 backend and convert
|
||||
// it into S3 compatible bucket policy info.
|
||||
func (l *b2Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
|
||||
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return policyInfo, err
|
||||
}
|
||||
if bkt.Type == bucketTypeReadOnly {
|
||||
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
|
||||
return policyInfo, nil
|
||||
}
|
||||
// bkt.Type can also be snapshot, but it is only allowed through B2 browser console,
|
||||
// just return back as policy not found for all cases.
|
||||
// CreateBucket always sets the value to allPrivate by default.
|
||||
return policy.BucketAccessPolicy{}, traceError(PolicyNotFound{Bucket: bucket})
|
||||
}
|
||||
|
||||
// DeleteBucketPolicies - resets the bucketType of bucket on B2 to 'allPrivate'.
|
||||
func (l *b2Objects) DeleteBucketPolicies(bucket string) error {
|
||||
bkt, err := l.Bucket(bucket)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bkt.Type = bucketTypePrivate
|
||||
_, err = bkt.Update(l.ctx)
|
||||
return b2ToObjectError(traceError(err))
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests headerToObjectInfo
|
||||
func TestHeaderToObjectInfo(t *testing.T) {
|
||||
testCases := []struct {
|
||||
bucket, object string
|
||||
header http.Header
|
||||
objInfo ObjectInfo
|
||||
}{
|
||||
{
|
||||
bucket: "bucket",
|
||||
object: "object",
|
||||
header: http.Header{
|
||||
"Content-Length": []string{"10"},
|
||||
"Content-Type": []string{"application/javascript"},
|
||||
"X-Bz-Upload-Timestamp": []string{"1000"},
|
||||
"X-Bz-Info-X-Amz-Meta-1": []string{"test1"},
|
||||
"X-Bz-File-Id": []string{"xxxxx"},
|
||||
},
|
||||
objInfo: ObjectInfo{
|
||||
Bucket: "bucket",
|
||||
Name: "object",
|
||||
ContentType: "application/javascript",
|
||||
Size: 10,
|
||||
UserDefined: map[string]string{
|
||||
"X-Amz-Meta-1": "test1",
|
||||
},
|
||||
ETag: "xxxxx",
|
||||
},
|
||||
},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
gotObjInfo, err := headerToObjectInfo(testCase.bucket, testCase.object, testCase.header)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: %s", i+1, err)
|
||||
}
|
||||
if gotObjInfo.Bucket != testCase.objInfo.Bucket {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Bucket, gotObjInfo.Bucket)
|
||||
}
|
||||
if gotObjInfo.Name != testCase.objInfo.Name {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.Name, gotObjInfo.Name)
|
||||
}
|
||||
if gotObjInfo.ContentType != testCase.objInfo.ContentType {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ContentType, gotObjInfo.ContentType)
|
||||
}
|
||||
if gotObjInfo.ETag != testCase.objInfo.ETag {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.objInfo.ETag, gotObjInfo.ETag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests mkRange test.
|
||||
func TestMkRange(t *testing.T) {
|
||||
testCases := []struct {
|
||||
offset, size int64
|
||||
expectedRng string
|
||||
}{
|
||||
// No offset set, size not set.
|
||||
{
|
||||
offset: 0,
|
||||
size: 0,
|
||||
expectedRng: "",
|
||||
},
|
||||
// Offset set, size not set.
|
||||
{
|
||||
offset: 10,
|
||||
size: 0,
|
||||
expectedRng: "bytes=10-",
|
||||
},
|
||||
// Offset set, size set.
|
||||
{
|
||||
offset: 10,
|
||||
size: 11,
|
||||
expectedRng: "bytes=10-20",
|
||||
},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
gotRng := mkRange(testCase.offset, testCase.size)
|
||||
if gotRng != testCase.expectedRng {
|
||||
t.Errorf("Test %d: expected %s, got %s", i+1, testCase.expectedRng, gotRng)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,12 +28,6 @@ func toGCSPublicURL(bucket, object string) string {
|
||||
return fmt.Sprintf("https://storage.googleapis.com/%s/%s", bucket, object)
|
||||
}
|
||||
|
||||
// AnonPutObject creates a new object anonymously with the incoming data,
|
||||
func (l *gcsGateway) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) {
|
||||
|
||||
return ObjectInfo{}, NotImplemented{}
|
||||
}
|
||||
|
||||
// AnonGetObject - Get object anonymously
|
||||
func (l *gcsGateway) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
req, err := http.NewRequest("GET", toGCSPublicURL(bucket, object), nil)
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (l *gcsGateway) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (l *gcsGateway) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return []BucketInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (l *gcsGateway) HealObject(bucket string, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (l *gcsGateway) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
return ListObjectsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (l *gcsGateway) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
return ListMultipartsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
+169
-94
@@ -19,24 +19,33 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"math"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/oauth2/google"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/iterator"
|
||||
|
||||
"github.com/minio/cli"
|
||||
minio "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"google.golang.org/api/googleapi"
|
||||
"google.golang.org/api/iterator"
|
||||
"google.golang.org/api/option"
|
||||
)
|
||||
|
||||
var (
|
||||
// Project ID format is not valid.
|
||||
errGCSInvalidProjectID = errors.New("GCS project id is either empty or invalid")
|
||||
|
||||
// Project ID not found
|
||||
errGCSProjectIDNotFound = errors.New("unknown project id")
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -59,16 +68,93 @@ const (
|
||||
// Refer https://cloud.google.com/storage/docs/composite-objects
|
||||
gcsMaxComponents = 32
|
||||
|
||||
// gcsMaxPartCount - maximum multipart parts GCS supports which is 32 x 32 = 1024.
|
||||
gcsMaxPartCount = 1024
|
||||
|
||||
// Every 24 hours we scan minio.sys.tmp to delete expired multiparts in minio.sys.tmp
|
||||
gcsCleanupInterval = time.Hour * 24
|
||||
|
||||
// The cleanup routine deletes files older than 2 weeks in minio.sys.tmp
|
||||
gcsMultipartExpiry = time.Hour * 24 * 14
|
||||
|
||||
// Project ID key in credentials.json
|
||||
gcsProjectIDKey = "project_id"
|
||||
|
||||
gcsBackend = "gcs"
|
||||
)
|
||||
|
||||
func init() {
|
||||
const gcsGatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [PROJECTID]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
PROJECTID:
|
||||
GCS project-id should be provided if GOOGLE_APPLICATION_CREDENTIALS environmental variable is not set.
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of GCS.
|
||||
MINIO_SECRET_KEY: Password or secret key of GCS.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
GCS credentials file:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: Path to credentials.json
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for GCS backend.
|
||||
$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
|
||||
(Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}} mygcsprojectid
|
||||
|
||||
`
|
||||
|
||||
MustRegisterGatewayCommand(cli.Command{
|
||||
Name: gcsBackend,
|
||||
Usage: "Google Cloud Storage.",
|
||||
Action: gcsGatewayMain,
|
||||
CustomHelpTemplate: gcsGatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway gcs' command line.
|
||||
func gcsGatewayMain(ctx *cli.Context) {
|
||||
projectID := ctx.Args().First()
|
||||
if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
|
||||
errorIf(errGCSProjectIDNotFound, "project-id should be provided as argument or GOOGLE_APPLICATION_CREDENTIALS should be set with path to credentials.json")
|
||||
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
||||
}
|
||||
if projectID != "" && !isValidGCSProjectIDFormat(projectID) {
|
||||
errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
|
||||
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
||||
}
|
||||
|
||||
startGateway(ctx, &GCSGateway{projectID})
|
||||
}
|
||||
|
||||
// GCSGateway implements Gateway.
|
||||
type GCSGateway struct {
|
||||
projectID string
|
||||
}
|
||||
|
||||
// Name returns the name of gcs gatewaylayer.
|
||||
func (g *GCSGateway) Name() string {
|
||||
return gcsBackend
|
||||
}
|
||||
|
||||
// NewGatewayLayer returns gcs gatewaylayer.
|
||||
func (g *GCSGateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
||||
return newGCSGatewayLayer(g.projectID)
|
||||
}
|
||||
|
||||
// Stored in gcs.json - Contents of this file is not used anywhere. It can be
|
||||
// used for debugging purposes.
|
||||
type gcsMultipartMetaV1 struct {
|
||||
@@ -105,12 +191,16 @@ func gcsToObjectError(err error, params ...string) error {
|
||||
|
||||
bucket := ""
|
||||
object := ""
|
||||
uploadID := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
if len(params) == 3 {
|
||||
uploadID = params[2]
|
||||
}
|
||||
|
||||
// in some cases just a plain error is being returned
|
||||
switch err.Error() {
|
||||
@@ -121,9 +211,15 @@ func gcsToObjectError(err error, params ...string) error {
|
||||
e.e = err
|
||||
return e
|
||||
case "storage: object doesn't exist":
|
||||
err = ObjectNotFound{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
if uploadID != "" {
|
||||
err = InvalidUploadID{
|
||||
UploadID: uploadID,
|
||||
}
|
||||
} else {
|
||||
err = ObjectNotFound{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
}
|
||||
e.e = err
|
||||
return e
|
||||
@@ -199,49 +295,9 @@ func isValidGCSProjectIDFormat(projectID string) bool {
|
||||
return gcsProjectIDRegex.MatchString(projectID)
|
||||
}
|
||||
|
||||
// checkGCSProjectID - checks if the project ID does really exist using resource manager API.
|
||||
func checkGCSProjectID(ctx context.Context, projectID string) error {
|
||||
// Check if a project id associated to the current account does really exist
|
||||
resourceManagerClient, err := google.DefaultClient(ctx, cloudresourcemanager.CloudPlatformReadOnlyScope)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
baseSvc, err := cloudresourcemanager.New(resourceManagerClient)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
projectSvc := cloudresourcemanager.NewProjectsService(baseSvc)
|
||||
|
||||
curPageToken := ""
|
||||
|
||||
// Iterate over projects list result pages and immediately return nil when
|
||||
// the project ID is found.
|
||||
for {
|
||||
resp, err := projectSvc.List().PageToken(curPageToken).Context(ctx).Do()
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error getting projects list: %s", err.Error())
|
||||
}
|
||||
|
||||
for _, p := range resp.Projects {
|
||||
if p.ProjectId == projectID {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
if resp.NextPageToken != "" {
|
||||
curPageToken = resp.NextPageToken
|
||||
} else {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return errGCSProjectIDNotFound
|
||||
}
|
||||
|
||||
// gcsGateway - Implements gateway for Minio and GCS compatible object storage servers.
|
||||
type gcsGateway struct {
|
||||
gatewayUnsupported
|
||||
client *storage.Client
|
||||
anonClient *minio.Core
|
||||
projectID string
|
||||
@@ -250,17 +306,37 @@ type gcsGateway struct {
|
||||
|
||||
const googleStorageEndpoint = "storage.googleapis.com"
|
||||
|
||||
// newGCSGateway returns gcs gatewaylayer
|
||||
func newGCSGateway(projectID string) (GatewayLayer, error) {
|
||||
// Returns projectID from the GOOGLE_APPLICATION_CREDENTIALS file.
|
||||
func gcsParseProjectID(credsFile string) (projectID string, err error) {
|
||||
contents, err := ioutil.ReadFile(credsFile)
|
||||
if err != nil {
|
||||
return projectID, err
|
||||
}
|
||||
googleCreds := make(map[string]string)
|
||||
if err = json.Unmarshal(contents, &googleCreds); err != nil {
|
||||
return projectID, err
|
||||
}
|
||||
return googleCreds[gcsProjectIDKey], err
|
||||
}
|
||||
|
||||
// newGCSGatewayLayer returns gcs gatewaylayer
|
||||
func newGCSGatewayLayer(projectID string) (GatewayLayer, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
err := checkGCSProjectID(ctx, projectID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
var err error
|
||||
if projectID == "" {
|
||||
// If project ID is not provided on command line, we figure it out
|
||||
// from the credentials.json file.
|
||||
projectID, err = gcsParseProjectID(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize a GCS client.
|
||||
client, err := storage.NewClient(ctx)
|
||||
// Send user-agent in this format for Google to obtain usage insights while participating in the
|
||||
// Google Cloud Technology Partners (https://cloud.google.com/partners/)
|
||||
client, err := storage.NewClient(ctx, option.WithUserAgent(fmt.Sprintf("Minio/%s (GPN:Minio;)", Version)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -546,7 +622,7 @@ func (l *gcsGateway) ListObjects(bucket string, prefix string, marker string, de
|
||||
Bucket: attrs.Bucket,
|
||||
ModTime: attrs.Updated,
|
||||
Size: attrs.Size,
|
||||
ETag: fmt.Sprintf("%d", attrs.CRC32C),
|
||||
ETag: toS3ETag(fmt.Sprintf("%d", attrs.CRC32C)),
|
||||
UserDefined: attrs.Metadata,
|
||||
ContentType: attrs.ContentType,
|
||||
ContentEncoding: attrs.ContentEncoding,
|
||||
@@ -688,7 +764,7 @@ func fromGCSAttrsToObjectInfo(attrs *storage.ObjectAttrs) ObjectInfo {
|
||||
Bucket: attrs.Bucket,
|
||||
ModTime: attrs.Updated,
|
||||
Size: attrs.Size,
|
||||
ETag: fmt.Sprintf("%d", attrs.CRC32C),
|
||||
ETag: toS3ETag(fmt.Sprintf("%d", attrs.CRC32C)),
|
||||
UserDefined: attrs.Metadata,
|
||||
ContentType: attrs.ContentType,
|
||||
ContentEncoding: attrs.ContentEncoding,
|
||||
@@ -712,19 +788,13 @@ func (l *gcsGateway) GetObjectInfo(bucket string, object string) (ObjectInfo, er
|
||||
}
|
||||
|
||||
// PutObject - Create a new object with the incoming data,
|
||||
func (l *gcsGateway) PutObject(bucket string, key string, data *HashReader, metadata map[string]string) (ObjectInfo, error) {
|
||||
|
||||
func (l *gcsGateway) PutObject(bucket string, key string, data *hash.Reader, metadata map[string]string) (ObjectInfo, error) {
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
if _, err := hex.DecodeString(metadata["etag"]); err != nil {
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
delete(metadata, "etag")
|
||||
|
||||
object := l.client.Bucket(bucket).Object(key)
|
||||
|
||||
w := object.NewWriter(l.ctx)
|
||||
@@ -735,17 +805,13 @@ func (l *gcsGateway) PutObject(bucket string, key string, data *HashReader, meta
|
||||
|
||||
if _, err := io.Copy(w, data); err != nil {
|
||||
// Close the object writer upon error.
|
||||
w.Close()
|
||||
w.CloseWithError(err)
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
|
||||
// Close the object writer upon success.
|
||||
w.Close()
|
||||
|
||||
if err := data.Verify(); err != nil { // Verify sha256sum after close.
|
||||
object.Delete(l.ctx)
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
|
||||
attrs, err := object.Attrs(l.ctx)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(err), bucket, key)
|
||||
@@ -818,27 +884,22 @@ func (l *gcsGateway) ListMultipartUploads(bucket string, prefix string, keyMarke
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CopyObjectPart - copy part of object to other bucket and object
|
||||
func (l *gcsGateway) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return PartInfo{}, traceError(NotSupported{})
|
||||
}
|
||||
|
||||
// Checks if minio.sys.tmp/multipart/v1/<upload-id>/gcs.json exists, returns
|
||||
// an object layer compatible error upon any error.
|
||||
func (l *gcsGateway) checkUploadIDExists(bucket string, key string, uploadID string) error {
|
||||
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx)
|
||||
return gcsToObjectError(traceError(err), bucket, key)
|
||||
return gcsToObjectError(traceError(err), bucket, key, uploadID)
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (l *gcsGateway) PutObjectPart(bucket string, key string, uploadID string, partNumber int, data *HashReader) (PartInfo, error) {
|
||||
func (l *gcsGateway) PutObjectPart(bucket string, key string, uploadID string, partNumber int, data *hash.Reader) (PartInfo, error) {
|
||||
if err := l.checkUploadIDExists(bucket, key, uploadID); err != nil {
|
||||
return PartInfo{}, err
|
||||
}
|
||||
etag := data.md5Sum
|
||||
etag := data.MD5HexString()
|
||||
if etag == "" {
|
||||
// Generate random ETag.
|
||||
etag = getMD5Hash([]byte(mustGetUUID()))
|
||||
etag = genETag()
|
||||
}
|
||||
object := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, partNumber, etag))
|
||||
w := object.NewWriter(l.ctx)
|
||||
@@ -852,11 +913,6 @@ func (l *gcsGateway) PutObjectPart(bucket string, key string, uploadID string, p
|
||||
}
|
||||
// Make sure to close the object writer upon success.
|
||||
w.Close()
|
||||
|
||||
if err := data.Verify(); err != nil {
|
||||
object.Delete(l.ctx)
|
||||
return PartInfo{}, gcsToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
return PartInfo{
|
||||
PartNumber: partNumber,
|
||||
ETag: etag,
|
||||
@@ -911,13 +967,13 @@ func (l *gcsGateway) AbortMultipartUpload(bucket string, key string, uploadID st
|
||||
// to the number of components you can compose per second. This rate counts both the
|
||||
// components being appended to a composite object as well as the components being
|
||||
// copied when the composite object of which they are a part is copied.
|
||||
func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID string, uploadedParts []completePart) (ObjectInfo, error) {
|
||||
func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID string, uploadedParts []CompletePart) (ObjectInfo, error) {
|
||||
meta := gcsMultipartMetaName(uploadID)
|
||||
object := l.client.Bucket(bucket).Object(meta)
|
||||
|
||||
partZeroAttrs, err := object.Attrs(l.ctx)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(err), bucket, key)
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(err), bucket, key, uploadID)
|
||||
}
|
||||
|
||||
r, err := object.NewReader(l.ctx)
|
||||
@@ -944,9 +1000,26 @@ func (l *gcsGateway) CompleteMultipartUpload(bucket string, key string, uploadID
|
||||
}
|
||||
|
||||
var parts []*storage.ObjectHandle
|
||||
for _, uploadedPart := range uploadedParts {
|
||||
partSizes := make([]int64, len(uploadedParts))
|
||||
for i, uploadedPart := range uploadedParts {
|
||||
parts = append(parts, l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID,
|
||||
uploadedPart.PartNumber, uploadedPart.ETag)))
|
||||
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(l.ctx)
|
||||
if pErr != nil {
|
||||
return ObjectInfo{}, gcsToObjectError(traceError(pErr), bucket, key, uploadID)
|
||||
}
|
||||
partSizes[i] = partAttr.Size
|
||||
}
|
||||
|
||||
// Error out if parts except last part sizing < 5MiB.
|
||||
for i, size := range partSizes[:len(partSizes)-1] {
|
||||
if size < globalMinPartSize {
|
||||
return ObjectInfo{}, traceError(PartTooSmall{
|
||||
PartNumber: uploadedParts[i].PartNumber,
|
||||
PartSize: size,
|
||||
PartETag: uploadedParts[i].ETag,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Returns name of the composed object.
|
||||
@@ -1044,7 +1117,6 @@ func (l *gcsGateway) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy
|
||||
if err != nil {
|
||||
return policy.BucketAccessPolicy{}, gcsToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
|
||||
for _, r := range rules {
|
||||
if r.Entity != storage.AllUsers || r.Role == storage.RoleOwner {
|
||||
@@ -1057,7 +1129,10 @@ func (l *gcsGateway) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy
|
||||
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyWriteOnly, bucket, "")
|
||||
}
|
||||
}
|
||||
|
||||
// Return NoSuchBucketPolicy error, when policy is not set
|
||||
if len(policyInfo.Statements) == 0 {
|
||||
return policy.BucketAccessPolicy{}, gcsToObjectError(traceError(PolicyNotFound{}), bucket)
|
||||
}
|
||||
return policyInfo, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
@@ -179,3 +181,30 @@ func TestFromMinioClientListBucketResultToV2Info(t *testing.T) {
|
||||
t.Errorf("fromMinioClientListBucketResultToV2Info() = %v, want %v", got, listBucketV2Info)
|
||||
}
|
||||
}
|
||||
|
||||
// Test for gcsParseProjectID
|
||||
func TestGCSParseProjectID(t *testing.T) {
|
||||
f, err := ioutil.TempFile("", "")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
|
||||
contents := `
|
||||
{
|
||||
"type": "service_account",
|
||||
"project_id": "miniotesting"
|
||||
}
|
||||
`
|
||||
f.WriteString(contents)
|
||||
projectID, err := gcsParseProjectID(f.Name())
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
if projectID != "miniotesting" {
|
||||
t.Errorf(`Expected projectID value to be "miniotesting"`)
|
||||
}
|
||||
}
|
||||
|
||||
+35
-42
@@ -18,7 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
goioutil "io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"strconv"
|
||||
@@ -28,6 +28,8 @@ import (
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
)
|
||||
|
||||
// GetObjectHandler - GET Object
|
||||
@@ -117,32 +119,19 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
startOffset = hrange.offsetBegin
|
||||
length = hrange.getLength()
|
||||
}
|
||||
// Indicates if any data was written to the http.ResponseWriter
|
||||
dataWritten := false
|
||||
// io.Writer type which keeps track if any data was written.
|
||||
writer := funcToWriter(func(p []byte) (int, error) {
|
||||
if !dataWritten {
|
||||
// Set headers on the first write.
|
||||
// Set standard object headers.
|
||||
setObjectHeaders(w, objInfo, hrange)
|
||||
|
||||
// Set any additional requested response headers.
|
||||
setHeadGetRespHeaders(w, r.URL.Query())
|
||||
|
||||
dataWritten = true
|
||||
}
|
||||
return w.Write(p)
|
||||
})
|
||||
|
||||
getObject := objectAPI.GetObject
|
||||
if reqAuthType == authTypeAnonymous {
|
||||
getObject = objectAPI.AnonGetObject
|
||||
}
|
||||
|
||||
setObjectHeaders(w, objInfo, hrange)
|
||||
setHeadGetRespHeaders(w, r.URL.Query())
|
||||
httpWriter := ioutil.WriteOnClose(w)
|
||||
// Reads the object at startOffset and writes to mw.
|
||||
if err = getObject(bucket, object, startOffset, length, writer); err != nil {
|
||||
if err = getObject(bucket, object, startOffset, length, httpWriter); err != nil {
|
||||
errorIf(err, "Unable to write to client.")
|
||||
if !dataWritten {
|
||||
if !httpWriter.HasWritten() {
|
||||
// Error response only if no data has been written to client yet. i.e if
|
||||
// partial data has already been written before an error
|
||||
// occurred then no point in setting StatusCode and
|
||||
@@ -151,11 +140,11 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
return
|
||||
}
|
||||
if !dataWritten {
|
||||
// If ObjectAPI.GetObject did not return error and no data has
|
||||
// been written it would mean that it is a 0-byte object.
|
||||
// call wrter.Write(nil) to set appropriate headers.
|
||||
writer.Write(nil)
|
||||
if err = httpWriter.Close(); err != nil {
|
||||
if !httpWriter.HasWritten() { // write error response only if no data has been written to client yet
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Get host and port from Request.RemoteAddr.
|
||||
@@ -257,9 +246,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure we hex encode md5sum here.
|
||||
metadata["etag"] = hex.EncodeToString(md5Bytes)
|
||||
|
||||
// Lock the object.
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, object)
|
||||
if objectLock.GetLock(globalOperationTimeout) != nil {
|
||||
@@ -268,20 +254,30 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
defer objectLock.Unlock()
|
||||
|
||||
var objInfo ObjectInfo
|
||||
var (
|
||||
// Make sure we hex encode md5sum here.
|
||||
md5hex = hex.EncodeToString(md5Bytes)
|
||||
sha256hex = ""
|
||||
putObject = objectAPI.PutObject
|
||||
reader = r.Body
|
||||
)
|
||||
|
||||
switch reqAuthType {
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
return
|
||||
case authTypeAnonymous:
|
||||
// Create anonymous object.
|
||||
objInfo, err = objectAPI.AnonPutObject(bucket, object, size, r.Body, metadata, "")
|
||||
putObject = objectAPI.AnonPutObject
|
||||
case authTypeStreamingSigned:
|
||||
// Initialize stream signature verifier.
|
||||
reader, s3Error := newSignV4ChunkedReader(r)
|
||||
var s3Error APIErrorCode
|
||||
reader, s3Error = newSignV4ChunkedReader(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
objInfo, err = objectAPI.PutObject(bucket, object, NewHashReader(reader, size, "", ""), metadata)
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
@@ -289,27 +285,24 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
objInfo, err = objectAPI.PutObject(bucket, object, NewHashReader(r.Body, size, "", ""), metadata)
|
||||
case authTypePresigned, authTypeSigned:
|
||||
if s3Error := reqSignatureV4Verify(r, serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
sha256sum := ""
|
||||
if !skipContentSha256Cksum(r) {
|
||||
sha256sum = getContentSha256Cksum(r)
|
||||
sha256hex = getContentSha256Cksum(r)
|
||||
}
|
||||
}
|
||||
|
||||
// Create object.
|
||||
objInfo, err = objectAPI.PutObject(bucket, object, NewHashReader(r.Body, size, "", sha256sum), metadata)
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
objInfo, err := putObject(bucket, object, hashReader, metadata)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to save an object %s", r.URL.Path)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
@@ -467,7 +460,7 @@ func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *h
|
||||
// Read access policy up to maxAccessPolicySize.
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html
|
||||
// bucket policies are limited to 20KB in size, using a limit reader.
|
||||
policyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
|
||||
policyBytes, err := goioutil.ReadAll(io.LimitReader(r.Body, maxAccessPolicySize))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read from client.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
|
||||
+35
-203
@@ -31,167 +31,41 @@ import (
|
||||
miniohttp "github.com/minio/minio/pkg/http"
|
||||
)
|
||||
|
||||
const azureGatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENDPOINT:
|
||||
Azure server endpoint. Default ENDPOINT is https://core.windows.net
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of Azure storage.
|
||||
MINIO_SECRET_KEY: Password or secret key of Azure storage.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}} https://azure.example.com
|
||||
`
|
||||
|
||||
const s3GatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENDPOINT:
|
||||
S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of S3 storage.
|
||||
MINIO_SECRET_KEY: Password or secret key of S3 storage.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for S3 backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ {{.HelpName}} https://play.minio.io:9000
|
||||
`
|
||||
|
||||
const gcsGatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} PROJECTID
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
PROJECTID:
|
||||
GCS project id, there are no defaults this is mandatory.
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of GCS.
|
||||
MINIO_SECRET_KEY: Password or secret key of GCS.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for GCS backend.
|
||||
$ export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
|
||||
(Instructions to generate credentials : https://developers.google.com/identity/protocols/application-default-credentials)
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}} mygcsprojectid
|
||||
|
||||
`
|
||||
|
||||
var (
|
||||
azureBackendCmd = cli.Command{
|
||||
Name: "azure",
|
||||
Usage: "Microsoft Azure Blob Storage.",
|
||||
Action: azureGatewayMain,
|
||||
CustomHelpTemplate: azureGatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
|
||||
s3BackendCmd = cli.Command{
|
||||
Name: "s3",
|
||||
Usage: "Amazon Simple Storage Service (S3).",
|
||||
Action: s3GatewayMain,
|
||||
CustomHelpTemplate: s3GatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
gcsBackendCmd = cli.Command{
|
||||
Name: "gcs",
|
||||
Usage: "Google Cloud Storage.",
|
||||
Action: gcsGatewayMain,
|
||||
CustomHelpTemplate: gcsGatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
|
||||
gatewayCmd = cli.Command{
|
||||
Name: "gateway",
|
||||
Usage: "Start object storage gateway.",
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd, gcsBackendCmd},
|
||||
}
|
||||
)
|
||||
|
||||
// Represents the type of the gateway backend.
|
||||
type gatewayBackend string
|
||||
// Gateway represents a gateway backend.
|
||||
type Gateway interface {
|
||||
// Name returns the unique name of the gateway.
|
||||
Name() string
|
||||
// NewGatewayLayer returns a new gateway layer.
|
||||
NewGatewayLayer() (GatewayLayer, error)
|
||||
}
|
||||
|
||||
const (
|
||||
azureBackend gatewayBackend = "azure"
|
||||
s3Backend gatewayBackend = "s3"
|
||||
gcsBackend gatewayBackend = "gcs"
|
||||
// Add more backends here.
|
||||
)
|
||||
|
||||
// Initialize gateway layer depending on the backend type.
|
||||
// Supported backend types are
|
||||
//
|
||||
// - Azure Blob Storage.
|
||||
// - AWS S3.
|
||||
// - Google Cloud Storage.
|
||||
// - Add your favorite backend here.
|
||||
func newGatewayLayer(backendType gatewayBackend, arg string) (GatewayLayer, error) {
|
||||
switch backendType {
|
||||
case azureBackend:
|
||||
return newAzureLayer(arg)
|
||||
case s3Backend:
|
||||
return newS3Gateway(arg)
|
||||
case gcsBackend:
|
||||
// FIXME: The following print command is temporary and
|
||||
// will be removed when gcs is ready for production use.
|
||||
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
||||
return newGCSGateway(arg)
|
||||
// RegisterGatewayCommand registers a new command for gateway.
|
||||
func RegisterGatewayCommand(cmd cli.Command) error {
|
||||
// We should not have multiple subcommands with same name.
|
||||
for _, c := range gatewayCmd.Subcommands {
|
||||
if c.Name == cmd.Name {
|
||||
return fmt.Errorf("duplicate gateway: %s", cmd.Name)
|
||||
}
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
|
||||
gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustRegisterGatewayCommand is like RegisterGatewayCommand but panics instead of returning error.
|
||||
func MustRegisterGatewayCommand(cmd cli.Command) {
|
||||
if err := RegisterGatewayCommand(cmd); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Return endpoint.
|
||||
@@ -247,46 +121,8 @@ func validateGatewayArguments(serverAddr, endpointAddr string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway azure' command line.
|
||||
func azureGatewayMain(ctx *cli.Context) {
|
||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, "azure", 1)
|
||||
}
|
||||
|
||||
// Validate gateway arguments.
|
||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
|
||||
|
||||
gatewayMain(ctx, azureBackend)
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway s3' command line.
|
||||
func s3GatewayMain(ctx *cli.Context) {
|
||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, "s3", 1)
|
||||
}
|
||||
|
||||
// Validate gateway arguments.
|
||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), ctx.Args().First()), "Invalid argument")
|
||||
|
||||
gatewayMain(ctx, s3Backend)
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway gcs' command line
|
||||
func gcsGatewayMain(ctx *cli.Context) {
|
||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
||||
}
|
||||
|
||||
if !isValidGCSProjectIDFormat(ctx.Args().First()) {
|
||||
errorIf(errGCSInvalidProjectID, "Unable to start GCS gateway with %s", ctx.Args().First())
|
||||
cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
|
||||
}
|
||||
|
||||
gatewayMain(ctx, gcsBackend)
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway'.
|
||||
func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||
// Handler for 'minio gateway <name>'.
|
||||
func startGateway(ctx *cli.Context, gw Gateway) {
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
|
||||
if quietFlag {
|
||||
@@ -306,8 +142,10 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||
handleCommonEnvVars()
|
||||
|
||||
// Validate if we have access, secret set through environment.
|
||||
gatewayName := gw.Name()
|
||||
if !globalIsEnvCreds {
|
||||
fatalIf(fmt.Errorf("Access and Secret keys should be set through ENVs for backend [%s]", backendType), "")
|
||||
errorIf(errors.New("Access and secret keys not set"), "Access and Secret keys should be set through ENVs for backend [%s]", gatewayName)
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
// Create certs path.
|
||||
@@ -329,7 +167,11 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||
|
||||
initNSLock(false) // Enable local namespace lock.
|
||||
|
||||
newObject, err := newGatewayLayer(backendType, ctx.Args().First())
|
||||
if ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
newObject, err := gw.NewGatewayLayer()
|
||||
fatalIf(err, "Unable to initialize gateway layer")
|
||||
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
@@ -366,7 +208,6 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||
// invalid/unsupported signatures.
|
||||
setAuthHandler,
|
||||
// Add new handlers here.
|
||||
|
||||
}
|
||||
|
||||
globalHTTPServer = miniohttp.NewServer([]string{gatewayAddr}, registerHandlers(router, handlerFns...), globalTLSCertificate)
|
||||
@@ -385,21 +226,12 @@ func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
if !quietFlag {
|
||||
mode := ""
|
||||
switch gatewayBackend(backendType) {
|
||||
case azureBackend:
|
||||
mode = globalMinioModeGatewayAzure
|
||||
case gcsBackend:
|
||||
mode = globalMinioModeGatewayGCS
|
||||
case s3Backend:
|
||||
mode = globalMinioModeGatewayS3
|
||||
}
|
||||
|
||||
mode := globalMinioModeGatewayPrefix + gatewayName
|
||||
// Check update mode.
|
||||
checkUpdate(mode)
|
||||
|
||||
// Print gateway startup message.
|
||||
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), backendType)
|
||||
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
|
||||
}
|
||||
|
||||
handleSignals()
|
||||
|
||||
@@ -19,8 +19,31 @@ package cmd
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/cli"
|
||||
)
|
||||
|
||||
// Test RegisterGatewayCommand
|
||||
func TestRegisterGatewayCommand(t *testing.T) {
|
||||
var err error
|
||||
|
||||
cmd := cli.Command{Name: "test"}
|
||||
err = RegisterGatewayCommand(cmd)
|
||||
if err != nil {
|
||||
t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
|
||||
}
|
||||
|
||||
// Should returns 'duplicated' error
|
||||
err = RegisterGatewayCommand(cmd)
|
||||
if err == nil {
|
||||
t.Errorf("RegisterGatewayCommand twice with same name should return error")
|
||||
} else {
|
||||
if err.Error() != "duplicate gateway: test" {
|
||||
t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test parseGatewayEndpoint
|
||||
func TestParseGatewayEndpoint(t *testing.T) {
|
||||
testCases := []struct {
|
||||
|
||||
+69
-58
@@ -18,9 +18,11 @@ package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// GatewayLayer - Interface to implement gateway mode.
|
||||
@@ -30,7 +32,7 @@ type GatewayLayer interface {
|
||||
AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
|
||||
AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
|
||||
|
||||
AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error)
|
||||
AnonPutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (ObjectInfo, error)
|
||||
|
||||
SetBucketPolicies(string, policy.BucketAccessPolicy) error
|
||||
GetBucketPolicies(string) (policy.BucketAccessPolicy, error)
|
||||
@@ -61,69 +63,78 @@ func registerGatewayAPIRouter(mux *router.Router, gw GatewayLayer) {
|
||||
// API Router
|
||||
apiRouter := mux.NewRoute().PathPrefix("/").Subrouter()
|
||||
|
||||
// Bucket router
|
||||
bucket := apiRouter.PathPrefix("/{bucket}").Subrouter()
|
||||
var routers []*router.Router
|
||||
if globalDomainName != "" {
|
||||
routers = append(routers, apiRouter.Host("{bucket:.+}."+globalDomainName).Subrouter())
|
||||
}
|
||||
routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
|
||||
|
||||
/// Object operations
|
||||
// Object operations
|
||||
for _, bucket := range routers {
|
||||
/// Object operations
|
||||
|
||||
// HeadObject
|
||||
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler)
|
||||
// CopyObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// PutObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// ListObjectPxarts
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// NewMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}")
|
||||
// GetObject
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(api.GetObjectHandler)
|
||||
// CopyObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler)
|
||||
// PutObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(api.PutObjectHandler)
|
||||
// DeleteObject
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler)
|
||||
// HeadObject
|
||||
bucket.Methods("HEAD").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.HeadObjectHandler))
|
||||
// CopyObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// PutObjectPart
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectPartHandler)).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// ListObjectPxarts
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.ListObjectPartsHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.CompleteMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// NewMultipartUpload
|
||||
bucket.Methods("POST").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.NewMultipartUploadHandler)).Queries("uploads", "")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.AbortMultipartUploadHandler)).Queries("uploadId", "{uploadId:.*}")
|
||||
// GetObject
|
||||
bucket.Methods("GET").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.GetObjectHandler))
|
||||
// CopyObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(httpTraceAll(api.CopyObjectHandler))
|
||||
// PutObject
|
||||
bucket.Methods("PUT").Path("/{object:.+}").HandlerFunc(httpTraceHdrs(api.PutObjectHandler))
|
||||
// DeleteObject
|
||||
bucket.Methods("DELETE").Path("/{object:.+}").HandlerFunc(httpTraceAll(api.DeleteObjectHandler))
|
||||
|
||||
/// Bucket operations
|
||||
/// Bucket operations
|
||||
|
||||
// GetBucketLocation
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketLocationHandler).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "")
|
||||
// GetBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "")
|
||||
// ListenBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(api.ListenBucketNotificationHandler).Queries("events", "{events:.*}")
|
||||
// ListMultipartUploads
|
||||
bucket.Methods("GET").HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "")
|
||||
// ListObjectsV2
|
||||
bucket.Methods("GET").HandlerFunc(api.ListObjectsV2Handler).Queries("list-type", "2")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(api.ListObjectsV1Handler)
|
||||
// PutBucketPolicy
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "")
|
||||
// PutBucketNotification
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "")
|
||||
// PutBucket
|
||||
bucket.Methods("PUT").HandlerFunc(api.PutBucketHandler)
|
||||
// HeadBucket
|
||||
bucket.Methods("HEAD").HandlerFunc(api.HeadBucketHandler)
|
||||
// PostPolicy
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler)
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(api.DeleteMultipleObjectsHandler)
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods("DELETE").HandlerFunc(api.DeleteBucketHandler)
|
||||
// GetBucketLocation
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketLocationHandler)).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketPolicyHandler)).Queries("policy", "")
|
||||
// GetBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.GetBucketNotificationHandler)).Queries("notification", "")
|
||||
// ListenBucketNotification
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListenBucketNotificationHandler)).Queries("events", "{events:.*}")
|
||||
// ListMultipartUploads
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListMultipartUploadsHandler)).Queries("uploads", "")
|
||||
// ListObjectsV2
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV2Handler)).Queries("list-type", "2")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods("GET").HandlerFunc(httpTraceAll(api.ListObjectsV1Handler))
|
||||
// PutBucketPolicy
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketPolicyHandler)).Queries("policy", "")
|
||||
// PutBucketNotification
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketNotificationHandler)).Queries("notification", "")
|
||||
// PutBucket
|
||||
bucket.Methods("PUT").HandlerFunc(httpTraceAll(api.PutBucketHandler))
|
||||
// HeadBucket
|
||||
bucket.Methods("HEAD").HandlerFunc(httpTraceAll(api.HeadBucketHandler))
|
||||
// PostPolicy
|
||||
bucket.Methods("POST").HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(httpTraceAll(api.PostPolicyBucketHandler))
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods("POST").HandlerFunc(httpTraceAll(api.DeleteMultipleObjectsHandler)).Queries("delete", "")
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketPolicyHandler)).Queries("policy", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods("DELETE").HandlerFunc(httpTraceAll(api.DeleteBucketHandler))
|
||||
}
|
||||
|
||||
/// Root operation
|
||||
|
||||
// ListBuckets
|
||||
apiRouter.Methods("GET").HandlerFunc(api.ListBucketsHandler)
|
||||
apiRouter.Methods("GET").Path("/").HandlerFunc(httpTraceAll(api.ListBucketsHandler))
|
||||
|
||||
// If none of the routes match.
|
||||
apiRouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
}
|
||||
|
||||
@@ -17,35 +17,15 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"io"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// AnonPutObject creates a new object anonymously with the incoming data,
|
||||
func (l *s3Objects) AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, e error) {
|
||||
var sha256sumBytes []byte
|
||||
|
||||
var err error
|
||||
if sha256sum != "" {
|
||||
sha256sumBytes, err = hex.DecodeString(sha256sum)
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
var md5sumBytes []byte
|
||||
md5sum := metadata["etag"]
|
||||
if md5sum != "" {
|
||||
md5sumBytes, err = hex.DecodeString(md5sum)
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
delete(metadata, "etag")
|
||||
}
|
||||
|
||||
oi, err := l.anonClient.PutObject(bucket, object, data, size, md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata))
|
||||
func (l *s3Objects) AnonPutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, e error) {
|
||||
oi, err := l.anonClient.PutObject(bucket, object, data, data.Size(), data.MD5(), data.SHA256(), toMinioClientMetadata(metadata))
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
@@ -55,11 +35,11 @@ func (l *s3Objects) AnonPutObject(bucket string, object string, size int64, data
|
||||
|
||||
// AnonGetObject - Get object anonymously
|
||||
func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
|
||||
r := minio.NewGetReqHeaders()
|
||||
if err := r.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
opts := minio.GetObjectOptions{}
|
||||
if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
object, _, err := l.anonClient.GetObject(bucket, key, r)
|
||||
object, _, err := l.anonClient.GetObject(bucket, key, opts)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
@@ -75,8 +55,7 @@ func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64,
|
||||
|
||||
// AnonGetObjectInfo - Get object info anonymously
|
||||
func (l *s3Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, e error) {
|
||||
r := minio.NewHeadReqHeaders()
|
||||
oi, err := l.anonClient.StatObject(bucket, object, r)
|
||||
oi, err := l.anonClient.StatObject(bucket, object, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (l *s3Objects) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (l *s3Objects) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return []BucketInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (l *s3Objects) HealObject(bucket string, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (l *s3Objects) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (l *s3Objects) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
return lmi, traceError(NotImplemented{})
|
||||
}
|
||||
+117
-83
@@ -19,15 +19,86 @@ package cmd
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"encoding/hex"
|
||||
|
||||
"github.com/minio/cli"
|
||||
minio "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
const (
|
||||
s3Backend = "s3"
|
||||
)
|
||||
|
||||
func init() {
|
||||
const s3GatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENDPOINT:
|
||||
S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of S3 storage.
|
||||
MINIO_SECRET_KEY: Password or secret key of S3 storage.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for S3 backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ {{.HelpName}} https://play.minio.io:9000
|
||||
`
|
||||
|
||||
MustRegisterGatewayCommand(cli.Command{
|
||||
Name: s3Backend,
|
||||
Usage: "Amazon Simple Storage Service (S3).",
|
||||
Action: s3GatewayMain,
|
||||
CustomHelpTemplate: s3GatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway s3' command line.
|
||||
func s3GatewayMain(ctx *cli.Context) {
|
||||
// Validate gateway arguments.
|
||||
host := ctx.Args().First()
|
||||
// Validate gateway arguments.
|
||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||
|
||||
startGateway(ctx, &S3Gateway{host})
|
||||
}
|
||||
|
||||
// S3Gateway implements Gateway.
|
||||
type S3Gateway struct {
|
||||
host string
|
||||
}
|
||||
|
||||
// Name implements Gateway interface.
|
||||
func (g *S3Gateway) Name() string {
|
||||
return s3Backend
|
||||
}
|
||||
|
||||
// NewGatewayLayer returns s3 gatewaylayer.
|
||||
func (g *S3Gateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||
return newS3GatewayLayer(g.host)
|
||||
}
|
||||
|
||||
// s3ToObjectError converts Minio errors to minio object layer errors.
|
||||
func s3ToObjectError(err error, params ...string) error {
|
||||
if err == nil {
|
||||
@@ -85,7 +156,11 @@ func s3ToObjectError(err error, params ...string) error {
|
||||
Object: object,
|
||||
}
|
||||
case "XAmzContentSHA256Mismatch":
|
||||
err = SHA256Mismatch{}
|
||||
err = hash.SHA256Mismatch{}
|
||||
case "NoSuchUpload":
|
||||
err = InvalidUploadID{}
|
||||
case "EntityTooSmall":
|
||||
err = PartTooSmall{}
|
||||
}
|
||||
|
||||
e.e = err
|
||||
@@ -94,13 +169,13 @@ func s3ToObjectError(err error, params ...string) error {
|
||||
|
||||
// s3Objects implements gateway for Minio and S3 compatible object storage servers.
|
||||
type s3Objects struct {
|
||||
gatewayUnsupported
|
||||
Client *minio.Core
|
||||
anonClient *minio.Core
|
||||
}
|
||||
|
||||
// newS3Gateway returns s3 gatewaylayer
|
||||
func newS3Gateway(host string) (GatewayLayer, error) {
|
||||
|
||||
// newS3GatewayLayer returns s3 gatewaylayer
|
||||
func newS3GatewayLayer(host string) (GatewayLayer, error) {
|
||||
var err error
|
||||
var endpoint string
|
||||
var secure = true
|
||||
@@ -196,7 +271,7 @@ func (l *s3Objects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
func (l *s3Objects) ListBuckets() ([]BucketInfo, error) {
|
||||
buckets, err := l.Client.ListBuckets()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, s3ToObjectError(traceError(err))
|
||||
}
|
||||
|
||||
b := make([]BucketInfo, len(buckets))
|
||||
@@ -290,22 +365,20 @@ func fromMinioClientListBucketResult(bucket string, result minio.ListBucketResul
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
|
||||
r := minio.NewGetReqHeaders()
|
||||
|
||||
if length < 0 && length != -1 {
|
||||
return s3ToObjectError(traceError(errInvalidArgument), bucket, key)
|
||||
}
|
||||
|
||||
opts := minio.GetObjectOptions{}
|
||||
if startOffset >= 0 && length >= 0 {
|
||||
if err := r.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
}
|
||||
object, _, err := l.Client.GetObject(bucket, key, r)
|
||||
object, _, err := l.Client.GetObject(bucket, key, opts)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
|
||||
defer object.Close()
|
||||
|
||||
if _, err := io.Copy(writer, object); err != nil {
|
||||
@@ -333,8 +406,7 @@ func fromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
|
||||
|
||||
// GetObjectInfo reads object info and replies back ObjectInfo
|
||||
func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
|
||||
r := minio.NewHeadReqHeaders()
|
||||
oi, err := l.Client.StatObject(bucket, object, r)
|
||||
oi, err := l.Client.StatObject(bucket, object, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
@@ -343,17 +415,8 @@ func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectI
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data,
|
||||
func (l *s3Objects) PutObject(bucket string, object string, data *HashReader, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
sha256sumBytes, err := hex.DecodeString(data.sha256Sum)
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
md5sumBytes, err := hex.DecodeString(metadata["etag"])
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
delete(metadata, "etag")
|
||||
oi, err := l.Client.PutObject(bucket, object, data, data.Size(), md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata))
|
||||
func (l *s3Objects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5(), data.SHA256(), toMinioClientMetadata(metadata))
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
@@ -361,35 +424,17 @@ func (l *s3Objects) PutObject(bucket string, object string, data *HashReader, me
|
||||
return fromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
|
||||
// CopyObject copies a blob from source container to destination container.
|
||||
// CopyObject copies an object from source bucket to a destination bucket.
|
||||
func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
// Source object
|
||||
src := minio.NewSourceInfo(srcBucket, srcObject, nil)
|
||||
|
||||
// Destination object
|
||||
var xamzMeta = map[string]string{}
|
||||
for key := range metadata {
|
||||
for _, prefix := range userMetadataKeyPrefixes {
|
||||
if strings.HasPrefix(key, prefix) {
|
||||
xamzMeta[key] = metadata[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
dst, err := minio.NewDestinationInfo(dstBucket, dstObject, nil, xamzMeta)
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), dstBucket, dstObject)
|
||||
}
|
||||
|
||||
if err = l.Client.CopyObject(dst, src); err != nil {
|
||||
// Set this header such that following CopyObject() always sets the right metadata on the destination.
|
||||
// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
|
||||
// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
|
||||
// So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API.
|
||||
metadata["x-amz-metadata-directive"] = "REPLACE"
|
||||
if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, metadata); err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), srcBucket, srcObject)
|
||||
}
|
||||
|
||||
oi, err := l.GetObjectInfo(dstBucket, dstObject)
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), dstBucket, dstObject)
|
||||
}
|
||||
|
||||
return oi, nil
|
||||
return l.GetObjectInfo(dstBucket, dstObject)
|
||||
}
|
||||
|
||||
// DeleteObject deletes a blob in bucket
|
||||
@@ -402,9 +447,9 @@ func (l *s3Objects) DeleteObject(bucket string, object string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// fromMinioClientUploadMetadata converts ObjectMultipartInfo to uploadMetadata
|
||||
func fromMinioClientUploadMetadata(omi minio.ObjectMultipartInfo) uploadMetadata {
|
||||
return uploadMetadata{
|
||||
// fromMinioClientMultipartInfo converts ObjectMultipartInfo to MultipartInfo
|
||||
func fromMinioClientMultipartInfo(omi minio.ObjectMultipartInfo) MultipartInfo {
|
||||
return MultipartInfo{
|
||||
Object: omi.Key,
|
||||
UploadID: omi.UploadID,
|
||||
Initiated: omi.Initiated,
|
||||
@@ -413,10 +458,10 @@ func fromMinioClientUploadMetadata(omi minio.ObjectMultipartInfo) uploadMetadata
|
||||
|
||||
// fromMinioClientListMultipartsInfo converts minio ListMultipartUploadsResult to ListMultipartsInfo
|
||||
func fromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) ListMultipartsInfo {
|
||||
uploads := make([]uploadMetadata, len(lmur.Uploads))
|
||||
uploads := make([]MultipartInfo, len(lmur.Uploads))
|
||||
|
||||
for i, um := range lmur.Uploads {
|
||||
uploads[i] = fromMinioClientUploadMetadata(um)
|
||||
uploads[i] = fromMinioClientMultipartInfo(um)
|
||||
}
|
||||
|
||||
commonPrefixes := make([]string, len(lmur.CommonPrefixes))
|
||||
@@ -472,13 +517,11 @@ func toMinioClientMetadata(metadata map[string]string) map[string]string {
|
||||
func (l *s3Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
// Create PutObject options
|
||||
opts := minio.PutObjectOptions{UserMetadata: metadata}
|
||||
return l.Client.NewMultipartUpload(bucket, object, opts)
|
||||
}
|
||||
|
||||
// CopyObjectPart copy part of object to other bucket and object
|
||||
func (l *s3Objects) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
// FIXME: implement CopyObjectPart
|
||||
return PartInfo{}, traceError(NotImplemented{})
|
||||
uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
|
||||
if err != nil {
|
||||
return uploadID, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// fromMinioClientObjectPart converts minio ObjectPart to PartInfo
|
||||
@@ -492,20 +535,10 @@ func fromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *HashReader) (pi PartInfo, e error) {
|
||||
md5HexBytes, err := hex.DecodeString(data.md5Sum)
|
||||
func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) {
|
||||
info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5(), data.SHA256())
|
||||
if err != nil {
|
||||
return pi, err
|
||||
}
|
||||
|
||||
sha256sumBytes, err := hex.DecodeString(data.sha256Sum)
|
||||
if err != nil {
|
||||
return pi, err
|
||||
}
|
||||
|
||||
info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), md5HexBytes, sha256sumBytes)
|
||||
if err != nil {
|
||||
return pi, err
|
||||
return pi, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectPart(info), nil
|
||||
@@ -548,19 +581,20 @@ func (l *s3Objects) ListObjectParts(bucket string, object string, uploadID strin
|
||||
|
||||
// AbortMultipartUpload aborts a ongoing multipart upload
|
||||
func (l *s3Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error {
|
||||
return l.Client.AbortMultipartUpload(bucket, object, uploadID)
|
||||
err := l.Client.AbortMultipartUpload(bucket, object, uploadID)
|
||||
return s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// toMinioClientCompletePart converts completePart to minio CompletePart
|
||||
func toMinioClientCompletePart(part completePart) minio.CompletePart {
|
||||
// toMinioClientCompletePart converts CompletePart to minio CompletePart
|
||||
func toMinioClientCompletePart(part CompletePart) minio.CompletePart {
|
||||
return minio.CompletePart{
|
||||
ETag: part.ETag,
|
||||
PartNumber: part.PartNumber,
|
||||
}
|
||||
}
|
||||
|
||||
// toMinioClientCompleteParts converts []completePart to minio []CompletePart
|
||||
func toMinioClientCompleteParts(parts []completePart) []minio.CompletePart {
|
||||
// toMinioClientCompleteParts converts []CompletePart to minio []CompletePart
|
||||
func toMinioClientCompleteParts(parts []CompletePart) []minio.CompletePart {
|
||||
mparts := make([]minio.CompletePart, len(parts))
|
||||
for i, part := range parts {
|
||||
mparts[i] = toMinioClientCompletePart(part)
|
||||
@@ -569,7 +603,7 @@ func toMinioClientCompleteParts(parts []completePart) []minio.CompletePart {
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (l *s3Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []completePart) (oi ObjectInfo, e error) {
|
||||
func (l *s3Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, e error) {
|
||||
err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, toMinioClientCompleteParts(uploadedParts))
|
||||
if err != nil {
|
||||
return oi, s3ToObjectError(traceError(err), bucket, object)
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
func errResponse(code string) minio.ErrorResponse {
|
||||
return minio.ErrorResponse{
|
||||
Code: code,
|
||||
}
|
||||
}
|
||||
|
||||
func TestS3ToObjectError(t *testing.T) {
|
||||
testCases := []struct {
|
||||
inputErr error
|
||||
expectedErr error
|
||||
bucket, object string
|
||||
}{
|
||||
{
|
||||
inputErr: errResponse("BucketAlreadyOwnedByYou"),
|
||||
expectedErr: BucketAlreadyOwnedByYou{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("BucketNotEmpty"),
|
||||
expectedErr: BucketNotEmpty{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("InvalidBucketName"),
|
||||
expectedErr: BucketNameInvalid{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("NoSuchBucketPolicy"),
|
||||
expectedErr: PolicyNotFound{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("NoSuchBucket"),
|
||||
expectedErr: BucketNotFound{},
|
||||
},
|
||||
// with empty Object in minio.ErrorRepsonse, NoSuchKey
|
||||
// is interpreted as BucketNotFound
|
||||
{
|
||||
inputErr: errResponse("NoSuchKey"),
|
||||
expectedErr: BucketNotFound{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("NoSuchUpload"),
|
||||
expectedErr: InvalidUploadID{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("XMinioInvalidObjectName"),
|
||||
expectedErr: ObjectNameInvalid{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("AccessDenied"),
|
||||
expectedErr: PrefixAccessDenied{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("XAmzContentSHA256Mismatch"),
|
||||
expectedErr: hash.SHA256Mismatch{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("EntityTooSmall"),
|
||||
expectedErr: PartTooSmall{},
|
||||
},
|
||||
{
|
||||
inputErr: nil,
|
||||
expectedErr: nil,
|
||||
},
|
||||
// Special test case for NoSuchKey with object name
|
||||
{
|
||||
inputErr: minio.ErrorResponse{
|
||||
Code: "NoSuchKey",
|
||||
},
|
||||
expectedErr: ObjectNotFound{},
|
||||
bucket: "bucket",
|
||||
object: "obbject",
|
||||
},
|
||||
|
||||
// N B error values that aren't of expected types
|
||||
// should be left untouched.
|
||||
// Special test case for error that is not of type
|
||||
// minio.ErrorResponse
|
||||
{
|
||||
inputErr: errors.New("not a minio.ErrorResponse"),
|
||||
expectedErr: errors.New("not a minio.ErrorResponse"),
|
||||
},
|
||||
// Special test case for error value that is not of
|
||||
// type (*Error)
|
||||
{
|
||||
inputErr: errors.New("not a *Error"),
|
||||
expectedErr: errors.New("not a *Error"),
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
actualErr := s3ToObjectError(tc.inputErr, tc.bucket, tc.object)
|
||||
if e, ok := actualErr.(*Error); ok && e.e != tc.expectedErr {
|
||||
t.Errorf("Test case %d: Expected error %v but received error %v", i+1, tc.expectedErr, e.e)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -22,7 +22,7 @@ import (
|
||||
)
|
||||
|
||||
// Prints the formatted startup message.
|
||||
func printGatewayStartupMessage(apiEndPoints []string, backendType gatewayBackend) {
|
||||
func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
|
||||
strippedAPIEndpoints := stripStandardPorts(apiEndPoints)
|
||||
|
||||
// Prints credential.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
type gatewayUnsupported struct{}
|
||||
|
||||
// CopyObjectPart - Not implemented.
|
||||
func (a gatewayUnsupported) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string,
|
||||
partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return info, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (a gatewayUnsupported) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (a gatewayUnsupported) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return nil, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (a gatewayUnsupported) HealObject(bucket, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (a gatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (a gatewayUnsupported) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
|
||||
delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
return lmi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonListObjects - List objects anonymously
|
||||
func (a gatewayUnsupported) AnonListObjects(bucket string, prefix string, marker string, delimiter string,
|
||||
maxKeys int) (loi ListObjectsInfo, err error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonListObjectsV2 - List objects in V2 mode, anonymously
|
||||
func (a gatewayUnsupported) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
|
||||
fetchOwner bool, startAfter string) (loi ListObjectsV2Info, err error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetBucketInfo - Get bucket metadata anonymously.
|
||||
func (a gatewayUnsupported) AnonGetBucketInfo(bucket string) (bi BucketInfo, err error) {
|
||||
return bi, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonPutObject creates a new object anonymously with the incoming data,
|
||||
func (a gatewayUnsupported) AnonPutObject(bucket, object string, data *hash.Reader,
|
||||
metadata map[string]string) (ObjectInfo, error) {
|
||||
return ObjectInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
+34
-5
@@ -108,6 +108,40 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// ReservedMetadataPrefix is the prefix of a metadata key which
|
||||
// is reserved and for internal use only.
|
||||
const ReservedMetadataPrefix = "X-Minio-Internal-"
|
||||
|
||||
type reservedMetadataHandler struct {
|
||||
http.Handler
|
||||
}
|
||||
|
||||
func filterReservedMetadata(h http.Handler) http.Handler {
|
||||
return reservedMetadataHandler{h}
|
||||
}
|
||||
|
||||
// ServeHTTP fails if the request contains at least one reserved header which
|
||||
// would be treated as metadata.
|
||||
func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if containsReservedMetadata(r.Header) {
|
||||
writeErrorResponse(w, ErrUnsupportedMetadata, r.URL)
|
||||
return
|
||||
}
|
||||
h.Handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// containsReservedMetadata returns true if the http.Header contains
|
||||
// keys which are treated as metadata but are reserved for internal use
|
||||
// and must not set by clients
|
||||
func containsReservedMetadata(header http.Header) bool {
|
||||
for key := range header {
|
||||
if strings.HasPrefix(key, ReservedMetadataPrefix) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Reserved bucket.
|
||||
const (
|
||||
minioReservedBucket = "minio"
|
||||
@@ -406,11 +440,6 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
// A put method on path "/" doesn't make sense, ignore it.
|
||||
if r.Method == httpPUT && r.URL.Path == "/" && r.Header.Get(minioAdminOpHeader) == "" {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Serve HTTP.
|
||||
h.handler.ServeHTTP(w, r)
|
||||
|
||||
@@ -144,3 +144,38 @@ func TestIsHTTPHeaderSizeTooLarge(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var containsReservedMetadataTests = []struct {
|
||||
header http.Header
|
||||
shouldFail bool
|
||||
}{
|
||||
{
|
||||
header: http.Header{"X-Minio-Key": []string{"value"}},
|
||||
},
|
||||
{
|
||||
header: http.Header{ServerSideEncryptionIV: []string{"iv"}},
|
||||
shouldFail: true,
|
||||
},
|
||||
{
|
||||
header: http.Header{ServerSideEncryptionSealAlgorithm: []string{SSESealAlgorithmDareSha256}},
|
||||
shouldFail: true,
|
||||
},
|
||||
{
|
||||
header: http.Header{ServerSideEncryptionSealedKey: []string{"mac"}},
|
||||
shouldFail: true,
|
||||
},
|
||||
{
|
||||
header: http.Header{ReservedMetadataPrefix + "Key": []string{"value"}},
|
||||
shouldFail: true,
|
||||
},
|
||||
}
|
||||
|
||||
func TestContainsReservedMetadata(t *testing.T) {
|
||||
for i, test := range containsReservedMetadataTests {
|
||||
if contains := containsReservedMetadata(test.header); contains && !test.shouldFail {
|
||||
t.Errorf("Test %d: contains reserved header but should not fail", i)
|
||||
} else if !contains && test.shouldFail {
|
||||
t.Errorf("Test %d: does not contain reserved header but failed", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-4
@@ -25,6 +25,7 @@ import (
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/fatih/color"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
miniohttp "github.com/minio/minio/pkg/http"
|
||||
)
|
||||
|
||||
@@ -50,9 +51,7 @@ const (
|
||||
globalMinioModeFS = "mode-server-fs"
|
||||
globalMinioModeXL = "mode-server-xl"
|
||||
globalMinioModeDistXL = "mode-server-distributed-xl"
|
||||
globalMinioModeGatewayAzure = "mode-gateway-azure"
|
||||
globalMinioModeGatewayS3 = "mode-gateway-s3"
|
||||
globalMinioModeGatewayGCS = "mode-gateway-gcs"
|
||||
globalMinioModeGatewayPrefix = "mode-gateway-"
|
||||
|
||||
// globalMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API.
|
||||
globalMinioSysTmp = "minio.sys.tmp/"
|
||||
@@ -124,6 +123,9 @@ var (
|
||||
globalHTTPServerErrorCh = make(chan error)
|
||||
globalOSSignalCh = make(chan os.Signal, 1)
|
||||
|
||||
// Enable HTTP request/response headers and body logging.
|
||||
globalHTTPTrace bool
|
||||
|
||||
// List of admin peers.
|
||||
globalAdminPeers = adminPeers{}
|
||||
|
||||
@@ -141,9 +143,12 @@ var (
|
||||
// Time when object layer was initialized on start up.
|
||||
globalBootTime time.Time
|
||||
|
||||
globalActiveCred credential
|
||||
globalActiveCred auth.Credentials
|
||||
globalPublicCerts []*x509.Certificate
|
||||
globalXLObjCacheDisabled bool
|
||||
|
||||
globalIsEnvDomainName bool
|
||||
globalDomainName string // Root domain for virtual host style requests
|
||||
// Add new variable global values here.
|
||||
|
||||
globalListingTimeout = newDynamicTimeout( /*30*/ 600*time.Second /*5*/, 600*time.Second) // timeout for listing related ops
|
||||
|
||||
@@ -19,9 +19,13 @@ package cmd
|
||||
import (
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
httptracer "github.com/minio/minio/pkg/handlers"
|
||||
)
|
||||
|
||||
// Parses location constraint from the incoming reader.
|
||||
@@ -240,3 +244,46 @@ func extractPostPolicyFormValues(form *multipart.Form) (filePart io.ReadCloser,
|
||||
|
||||
return filePart, fileName, fileSize, formValues, nil
|
||||
}
|
||||
|
||||
// Log headers and body.
|
||||
func httpTraceAll(f http.HandlerFunc) http.HandlerFunc {
|
||||
if !globalHTTPTrace {
|
||||
return f
|
||||
}
|
||||
return httptracer.TraceReqHandlerFunc(f, os.Stdout, true)
|
||||
}
|
||||
|
||||
// Log only the headers.
|
||||
func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc {
|
||||
if !globalHTTPTrace {
|
||||
return f
|
||||
}
|
||||
return httptracer.TraceReqHandlerFunc(f, os.Stdout, false)
|
||||
}
|
||||
|
||||
// Returns "/bucketName/objectName" for path-style or virtual-host-style requests.
|
||||
func getResource(path string, host string, domain string) (string, error) {
|
||||
if domain == "" {
|
||||
return path, nil
|
||||
}
|
||||
// If virtual-host-style is enabled construct the "resource" properly.
|
||||
if strings.Contains(host, ":") {
|
||||
// In bucket.mydomain.com:9000, strip out :9000
|
||||
var err error
|
||||
if host, _, err = net.SplitHostPort(host); err != nil {
|
||||
errorIf(err, "Unable to split %s", host)
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
if !strings.HasSuffix(host, "."+domain) {
|
||||
return path, nil
|
||||
}
|
||||
bucket := strings.TrimSuffix(host, "."+domain)
|
||||
return slashSeparator + pathJoin(bucket, path), nil
|
||||
}
|
||||
|
||||
// If none of the http routes match respond with MethodNotAllowed
|
||||
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -190,3 +190,26 @@ func TestExtractMetadataHeaders(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test getResource()
|
||||
func TestGetResource(t *testing.T) {
|
||||
testCases := []struct {
|
||||
p string
|
||||
host string
|
||||
domain string
|
||||
expectedResource string
|
||||
}{
|
||||
{"/a/b/c", "test.mydomain.com", "mydomain.com", "/test/a/b/c"},
|
||||
{"/a/b/c", "test.mydomain.com", "notmydomain.com", "/a/b/c"},
|
||||
{"/a/b/c", "test.mydomain.com", "", "/a/b/c"},
|
||||
}
|
||||
for i, test := range testCases {
|
||||
gotResource, err := getResource(test.p, test.host, test.domain)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotResource != test.expectedResource {
|
||||
t.Fatalf("test %d: expected %s got %s", i+1, test.expectedResource, gotResource)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+5
-1
@@ -24,6 +24,7 @@ import (
|
||||
|
||||
jwtgo "github.com/dgrijalva/jwt-go"
|
||||
jwtreq "github.com/dgrijalva/jwt-go/request"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -47,7 +48,7 @@ var (
|
||||
)
|
||||
|
||||
func authenticateJWT(accessKey, secretKey string, expiry time.Duration) (string, error) {
|
||||
passedCredential, err := createCredential(accessKey, secretKey)
|
||||
passedCredential, err := auth.CreateCredentials(accessKey, secretKey)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
@@ -93,6 +94,9 @@ func keyFuncCallback(jwtToken *jwtgo.Token) (interface{}, error) {
|
||||
}
|
||||
|
||||
func isAuthTokenValid(tokenString string) bool {
|
||||
if tokenString == "" {
|
||||
return false
|
||||
}
|
||||
var claims jwtgo.StandardClaims
|
||||
jwtToken, err := jwtgo.ParseWithClaims(tokenString, &claims, keyFuncCallback)
|
||||
if err != nil {
|
||||
|
||||
+5
-7
@@ -20,6 +20,8 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
|
||||
func testAuthenticate(authType string, t *testing.T) {
|
||||
@@ -28,11 +30,7 @@ func testAuthenticate(authType string, t *testing.T) {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
defer os.RemoveAll(testPath)
|
||||
// Create access and secret keys in length, 300 and 600
|
||||
cred, err := getNewCredential(300, 600)
|
||||
if err != nil {
|
||||
t.Fatalf("unable to get new credential, %v", err)
|
||||
}
|
||||
cred := auth.MustGetNewCredentials()
|
||||
serverConfig.SetCredential(cred)
|
||||
|
||||
// Define test cases.
|
||||
@@ -42,9 +40,9 @@ func testAuthenticate(authType string, t *testing.T) {
|
||||
expectedErr error
|
||||
}{
|
||||
// Access key (less than 5 chrs) too small.
|
||||
{"user", cred.SecretKey, errInvalidAccessKeyLength},
|
||||
{"user", cred.SecretKey, auth.ErrInvalidAccessKeyLength},
|
||||
// Secret key (less than 8 chrs) too small.
|
||||
{cred.AccessKey, "pass", errInvalidSecretKeyLength},
|
||||
{cred.AccessKey, "pass", auth.ErrInvalidSecretKeyLength},
|
||||
// Authentication error.
|
||||
{"myuser", "mypassword", errInvalidAccessKeyID},
|
||||
// Authentication error.
|
||||
|
||||
+1
-10
@@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
@@ -258,13 +257,5 @@ func (n *nsLockMap) deleteLockInfoEntryForOps(param nsParam, opsID string) error
|
||||
|
||||
// Return randomly generated string ID
|
||||
func getOpsID() string {
|
||||
const opsIDLen = 16
|
||||
opsIDBytes := make([]byte, opsIDLen)
|
||||
if _, err := rand.Read(opsIDBytes); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for i := 0; i < opsIDLen; i++ {
|
||||
opsIDBytes[i] = alphaNumericTable[opsIDBytes[i]%alphaNumericTableLen]
|
||||
}
|
||||
return string(opsIDBytes)
|
||||
return mustGetUUID()
|
||||
}
|
||||
|
||||
@@ -38,84 +38,6 @@ type lockStateCase struct {
|
||||
expectedVolPathBlockCount int // Total locks blocked on the given <volume, path> pair.
|
||||
}
|
||||
|
||||
// Used for validating the Lock info obtaining from contol RPC end point for obtaining lock related info.
|
||||
func verifyRPCLockInfoResponse(l lockStateCase, rpcLockInfoMap map[string]*SystemLockState, t TestErrHandler, testNum int) {
|
||||
for _, rpcLockInfoResponse := range rpcLockInfoMap {
|
||||
// Assert the total number of locks (locked + acquired) in the system.
|
||||
if rpcLockInfoResponse.TotalLocks != int64(l.expectedGlobalLockCount) {
|
||||
t.Fatalf("Test %d: Expected the global lock counter to be %v, but got %v", testNum, int64(l.expectedGlobalLockCount),
|
||||
rpcLockInfoResponse.TotalLocks)
|
||||
}
|
||||
|
||||
// verify the count for total blocked locks.
|
||||
if rpcLockInfoResponse.TotalBlockedLocks != int64(l.expectedBlockedLockCount) {
|
||||
t.Fatalf("Test %d: Expected the total blocked lock counter to be %v, but got %v", testNum, int64(l.expectedBlockedLockCount),
|
||||
rpcLockInfoResponse.TotalBlockedLocks)
|
||||
}
|
||||
|
||||
// verify the count for total running locks.
|
||||
if rpcLockInfoResponse.TotalAcquiredLocks != int64(l.expectedRunningLockCount) {
|
||||
t.Fatalf("Test %d: Expected the total running lock counter to be %v, but got %v", testNum, int64(l.expectedRunningLockCount),
|
||||
rpcLockInfoResponse.TotalAcquiredLocks)
|
||||
}
|
||||
|
||||
for _, locksInfoPerObject := range rpcLockInfoResponse.LocksInfoPerObject {
|
||||
// See whether the entry for the <bucket, object> exists in the RPC response.
|
||||
if locksInfoPerObject.Bucket == l.volume && locksInfoPerObject.Object == l.path {
|
||||
// Assert the total number of locks (blocked + acquired) for the given <buckt, object> pair.
|
||||
if locksInfoPerObject.LocksOnObject != int64(l.expectedVolPathLockCount) {
|
||||
t.Errorf("Test %d: Expected the total lock count for bucket: \"%s\", object: \"%s\" to be %v, but got %v", testNum,
|
||||
l.volume, l.path, int64(l.expectedVolPathLockCount), locksInfoPerObject.LocksOnObject)
|
||||
}
|
||||
// Assert the total number of acquired locks for the given <buckt, object> pair.
|
||||
if locksInfoPerObject.LocksAcquiredOnObject != int64(l.expectedVolPathRunningCount) {
|
||||
t.Errorf("Test %d: Expected the acquired lock count for bucket: \"%s\", object: \"%s\" to be %v, but got %v", testNum,
|
||||
l.volume, l.path, int64(l.expectedVolPathRunningCount), locksInfoPerObject.LocksAcquiredOnObject)
|
||||
}
|
||||
// Assert the total number of blocked locks for the given <buckt, object> pair.
|
||||
if locksInfoPerObject.TotalBlockedLocks != int64(l.expectedVolPathBlockCount) {
|
||||
t.Errorf("Test %d: Expected the blocked lock count for bucket: \"%s\", object: \"%s\" to be %v, but got %v", testNum,
|
||||
l.volume, l.path, int64(l.expectedVolPathBlockCount), locksInfoPerObject.TotalBlockedLocks)
|
||||
}
|
||||
// Flag to mark whether there's an entry in the RPC lock info response for given opsID.
|
||||
var opsIDfound bool
|
||||
for _, opsLockState := range locksInfoPerObject.LockDetailsOnObject {
|
||||
// first check whether the entry for the given operation ID exists.
|
||||
if opsLockState.OperationID == l.opsID {
|
||||
opsIDfound = true
|
||||
// asserting the type of lock (RLock/WLock) from the RPC lock info response.
|
||||
if l.readLock {
|
||||
if opsLockState.LockType != debugRLockStr {
|
||||
t.Errorf("Test case %d: Expected the lock type to be \"%s\"", testNum, debugRLockStr)
|
||||
}
|
||||
} else {
|
||||
if opsLockState.LockType != debugWLockStr {
|
||||
t.Errorf("Test case %d: Expected the lock type to be \"%s\"", testNum, debugWLockStr)
|
||||
}
|
||||
}
|
||||
|
||||
if opsLockState.Status != l.expectedLockStatus {
|
||||
t.Errorf("Test case %d: Expected the status of the operation to be \"%s\", got \"%s\"", testNum, l.expectedLockStatus, opsLockState.Status)
|
||||
}
|
||||
|
||||
// all check satisfied, return here.
|
||||
// Any mismatch in the earlier checks would have ended the tests due to `Fatalf`,
|
||||
// control reaching here implies that all checks are satisfied.
|
||||
return
|
||||
}
|
||||
}
|
||||
// opsID not found.
|
||||
// No entry for an operation with given operation ID exists.
|
||||
if !opsIDfound {
|
||||
t.Fatalf("Test case %d: Entry for OpsId: \"%s\" not found in <bucket>: \"%s\", <path>: \"%s\" doesn't exist in the RPC response", testNum, l.opsID, l.volume, l.path)
|
||||
}
|
||||
}
|
||||
}
|
||||
// No entry exists for given <bucket, object> pair in the RPC response.
|
||||
t.Errorf("Test case %d: Entry for <bucket>: \"%s\", <object>: \"%s\" doesn't exist in the RPC response", testNum, l.volume, l.path)
|
||||
}
|
||||
}
|
||||
|
||||
// Read entire state of the locks in the system and return.
|
||||
func getSystemLockState() (SystemLockState, error) {
|
||||
globalNSMutex.lockMapMutex.Lock()
|
||||
@@ -257,25 +179,6 @@ func verifyLockState(l lockStateCase, t *testing.T, testNum int) {
|
||||
verifyLockStats(l, t, testNum)
|
||||
}
|
||||
|
||||
func TestGetOpsID(t *testing.T) {
|
||||
// Ensure that it returns an alphanumeric result of length 16.
|
||||
var id = getOpsID()
|
||||
|
||||
if len(id) != 16 {
|
||||
t.Fail()
|
||||
}
|
||||
|
||||
var e rune
|
||||
for _, char := range id {
|
||||
e = rune(char)
|
||||
|
||||
// Ensure that it is alphanumeric, in this case, between 0-9 and A-Z.
|
||||
if !(('0' <= e && e <= '9') || ('A' <= e && e <= 'Z')) {
|
||||
t.Fail()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewDebugLockInfoPerVolumePath - Validates the values initialized by newDebugLockInfoPerVolumePath().
|
||||
func TestNewDebugLockInfoPerVolumePath(t *testing.T) {
|
||||
lockInfo := &debugLockInfoPerVolumePath{
|
||||
@@ -296,6 +199,7 @@ func TestNewDebugLockInfoPerVolumePath(t *testing.T) {
|
||||
|
||||
// TestNsLockMapStatusBlockedToRunning - Validates the function for changing the lock state from blocked to running.
|
||||
func TestNsLockMapStatusBlockedToRunning(t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
volume string
|
||||
path string
|
||||
|
||||
+3
-2
@@ -190,10 +190,11 @@ func logIf(level logrus.Level, source string, err error, msg string, data ...int
|
||||
switch err.(type) {
|
||||
case BucketNotFound, BucketNotEmpty, BucketExists:
|
||||
ok = true
|
||||
case ObjectNotFound, ObjectExistsAsDirectory, BucketPolicyNotFound, InvalidUploadID, BadDigest:
|
||||
case ObjectNotFound, ObjectExistsAsDirectory:
|
||||
ok = true
|
||||
case BucketPolicyNotFound, InvalidUploadID:
|
||||
ok = true
|
||||
}
|
||||
|
||||
return ok
|
||||
}
|
||||
|
||||
|
||||
+2
-2
@@ -98,9 +98,9 @@ func newApp(name string) *cli.App {
|
||||
|
||||
// Register all commands.
|
||||
registerCommand(serverCmd)
|
||||
registerCommand(versionCmd)
|
||||
registerCommand(updateCmd)
|
||||
registerCommand(gatewayCmd)
|
||||
registerCommand(updateCmd)
|
||||
registerCommand(versionCmd)
|
||||
|
||||
// Set up app.
|
||||
cli.HelpFlag = cli.BoolFlag{
|
||||
|
||||
@@ -1,26 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "gopkg.in/check.v1"
|
||||
)
|
||||
|
||||
// Hook up gocheck into the "go test" runner.
|
||||
func Test(t *testing.T) { TestingT(t) }
|
||||
@@ -22,9 +22,10 @@ import (
|
||||
"sync"
|
||||
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/minio/dsync"
|
||||
"github.com/minio/lsync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Global name space lock.
|
||||
|
||||
@@ -175,6 +175,21 @@ func TestGetAPIEndpoints(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Ask the kernel for a free open port.
|
||||
func getFreePort() string {
|
||||
addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
l, err := net.ListenTCP("tcp", addr)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer l.Close()
|
||||
return fmt.Sprintf("%d", l.Addr().(*net.TCPAddr).Port)
|
||||
}
|
||||
|
||||
// Tests for port availability logic written for server startup sequence.
|
||||
func TestCheckPortAvailability(t *testing.T) {
|
||||
// Make a port is not available.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user