mirror of
https://github.com/pgsty/minio.git
synced 2026-08-08 23:33:30 +03:00
Compare commits
225 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ed1275a063 | |||
| a7d407fa42 | |||
| 4e6e05f8e0 | |||
| b0deea27df | |||
| e98d89274f | |||
| c59206bcd3 | |||
| 7f2d439baa | |||
| 5a80cbec2a | |||
| 2d19011a1d | |||
| e82dcd195c | |||
| fcb56d864c | |||
| 75cd4201b0 | |||
| f24c017e9a | |||
| b5280ba243 | |||
| 2a0e4b6f58 | |||
| 1898961ce3 | |||
| 236796ebd6 | |||
| 4e4f855b30 | |||
| 2db22deb93 | |||
| fb8d0d7cf7 | |||
| a536cf5dc0 | |||
| b9b68e9331 | |||
| 632022971b | |||
| def04f01cf | |||
| bc67410548 | |||
| 7881791a91 | |||
| d2f8f8c7ee | |||
| 8c32311b80 | |||
| 9bb88e610e | |||
| d1e41695fe | |||
| 2aeb3fbe86 | |||
| 99b843a64e | |||
| 4f31a9a33b | |||
| 65ddff8899 | |||
| e7c902bbbc | |||
| 5a5895203b | |||
| 7da0336ac8 | |||
| 3be616de3f | |||
| c5bf22fd90 | |||
| 7c9f934875 | |||
| b6f9b24b30 | |||
| 13cb814a0e | |||
| d264d2c899 | |||
| bebaff269c | |||
| 52b159b1db | |||
| 324834e4da | |||
| c2ed1347d9 | |||
| 50f6f9fe58 | |||
| 48cb0ea34b | |||
| 6f7c99a333 | |||
| 3498f5b0ec | |||
| 21d8c0fd13 | |||
| 79b9a9ce46 | |||
| b9b353db4b | |||
| 11a9b317a3 | |||
| 9af7d627ac | |||
| 76d9d54603 | |||
| 4c7c571875 | |||
| 313ba74b09 | |||
| 3e124315c8 | |||
| 78a0fd951e | |||
| 40852801ea | |||
| f6980c4630 | |||
| 8fcc787cba | |||
| 4e6d3c093f | |||
| 61145361fd | |||
| 950b4ad9af | |||
| 6add646130 | |||
| 20e61fb362 | |||
| 18ced1102c | |||
| d6af3c1237 | |||
| 5549a44566 | |||
| e7af31c2ff | |||
| e7971b1d55 | |||
| 26120d7838 | |||
| bef7c01c58 | |||
| 6a8ccc5925 | |||
| d85199e9de | |||
| 8608a84c23 | |||
| b7226f4c82 | |||
| f930ffe9e2 | |||
| b50a245208 | |||
| 45bb11e020 | |||
| b65cf281fd | |||
| f781548b0c | |||
| 25ee8e74f7 | |||
| c975d2cc7e | |||
| ea66528739 | |||
| e1164103d4 | |||
| 83fe70f710 | |||
| 12a6523fb2 | |||
| dba61867e8 | |||
| dd092f6c2b | |||
| 2a810c7da2 | |||
| dd8c2aa5c6 | |||
| 9e3fce441e | |||
| d4265f9a13 | |||
| 2fc024e880 | |||
| eddf468aef | |||
| b0d04b9a81 | |||
| a9de303d8b | |||
| 69bd6df464 | |||
| bfb505aa8e | |||
| d732b1ff9d | |||
| ef517bd0da | |||
| 32d837cf88 | |||
| 7b579caf68 | |||
| 272b8003d6 | |||
| 1c24c93f73 | |||
| 712abc7958 | |||
| 5f6d717b7a | |||
| 7e1661f4fa | |||
| f9779b24ad | |||
| f1f23f6f11 | |||
| cf26c937e4 | |||
| f19f957668 | |||
| d6572879a8 | |||
| b6ab8f50fa | |||
| c82acc599a | |||
| 2447bb58dd | |||
| a55a298e00 | |||
| 2929c1832d | |||
| aa2d8583ad | |||
| df2d75a2a3 | |||
| b24b320807 | |||
| c872c1f1dc | |||
| a40610d331 | |||
| 38978eb2aa | |||
| ca7c3a3278 | |||
| d58fc68137 | |||
| 71c66464c1 | |||
| bf414068a3 | |||
| 88959ce600 | |||
| 572719872d | |||
| bdea19b583 | |||
| eb1f9c9916 | |||
| d07fb41fe8 | |||
| a9cda850ca | |||
| bef0318c36 | |||
| 3f19ea98bb | |||
| c96073f985 | |||
| d2f240c791 | |||
| 6491dfbbd6 | |||
| d9cfa5fcd3 | |||
| 89b14639a9 | |||
| 3f744c0361 | |||
| 9ed7fb4916 | |||
| 4280e68de3 | |||
| f162d7bd97 | |||
| 36990aeafd | |||
| 9fe51e392b | |||
| 7e879a45d5 | |||
| 1c911c5f40 | |||
| bd8dc17b7a | |||
| 8491a29ec3 | |||
| 81d21850ec | |||
| c6ec3fdfba | |||
| 88c3dd49c6 | |||
| 6869f6d9dd | |||
| 3f643acb99 | |||
| ea73accefd | |||
| 555d54371c | |||
| bab4c90c45 | |||
| a2fc0b14d6 | |||
| bf66e9a529 | |||
| fde8c38638 | |||
| e6252dee5a | |||
| ecb042aa1c | |||
| e29009d347 | |||
| 9631d65552 | |||
| 7b7be66fa1 | |||
| b99aaab42e | |||
| 32bd1b31e9 | |||
| f287b15e71 | |||
| 586466584f | |||
| c0b4bf0a3e | |||
| acf46cc3b5 | |||
| 06ef8248c3 | |||
| 7c2ae4eaf7 | |||
| 989d7af9ac | |||
| 8a6c3aa3cd | |||
| 62b560510b | |||
| 0edfb32621 | |||
| 7e0f1eb8b5 | |||
| b43e8337b1 | |||
| 30040fba45 | |||
| 44cf9ac62f | |||
| 3b55357045 | |||
| 18d9a20ff6 | |||
| 6590aba6d2 | |||
| 2e81f27d27 | |||
| ae3c05aa37 | |||
| cef044178c | |||
| c998d1ac8c | |||
| 3457e504cf | |||
| 7f0cca075a | |||
| 088c595e01 | |||
| 26b4b466df | |||
| fdf691fdcc | |||
| 88c8c2d6cd | |||
| ef585037a0 | |||
| 362ebdcbab | |||
| b251454dd6 | |||
| 21c8693d9c | |||
| 1e7e5e297c | |||
| c7f180ffa9 | |||
| b8bd8d6a03 | |||
| baec331e84 | |||
| 557f382477 | |||
| 23b166b318 | |||
| 81a481e098 | |||
| 5b3090dffc | |||
| 3ef3fefd54 | |||
| 28e25eac78 | |||
| b0c9ae7490 | |||
| 143e7fe300 | |||
| f09e7ca764 | |||
| fae284d6b9 | |||
| 83d8e01c81 | |||
| 110458cd10 | |||
| e3eec89d24 | |||
| 54ae364def | |||
| 16a100b597 | |||
| cbc5d78a09 | |||
| d8a2975a68 |
+20
-9
@@ -1,10 +1,4 @@
|
||||
go_import_path: github.com/minio/minio
|
||||
sudo: required
|
||||
|
||||
services:
|
||||
- docker
|
||||
|
||||
dist: trusty
|
||||
|
||||
language: go
|
||||
|
||||
@@ -18,20 +12,37 @@ branches:
|
||||
matrix:
|
||||
include:
|
||||
- os: linux
|
||||
dist: trusty
|
||||
sudo: required
|
||||
env:
|
||||
- ARCH=x86_64
|
||||
go: 1.10.3
|
||||
- CGO_ENABLED=0
|
||||
go: 1.11.4
|
||||
script:
|
||||
- make
|
||||
- diff -au <(gofmt -s -d cmd) <(printf "")
|
||||
- diff -au <(gofmt -s -d pkg) <(printf "")
|
||||
- make test GOFLAGS="-timeout 15m -race -v"
|
||||
- for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 15m "$d"; done
|
||||
- make verify
|
||||
- make coverage
|
||||
- cd browser && yarn && yarn test && cd ..
|
||||
- os: windows
|
||||
env:
|
||||
- ARCH=x86_64
|
||||
- CGO_ENABLED=0
|
||||
go: 1.11.4
|
||||
script:
|
||||
- go build --ldflags="$(go run buildscripts/gen-ldflags.go)" -o %GOPATH%\bin\minio.exe
|
||||
- for d in $(go list ./... | grep -v browser); do CGO_ENABLED=1 go test -v -race --timeout 20m "$d"; done
|
||||
- bash buildscripts/go-coverage.sh
|
||||
|
||||
before_script:
|
||||
# Add an IPv6 config - see the corresponding Travis issue
|
||||
# https://github.com/travis-ci/travis-ci/issues/8361
|
||||
- if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
|
||||
|
||||
before_install:
|
||||
- nvm install stable
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
|
||||
|
||||
after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
+15
-11
@@ -1,28 +1,32 @@
|
||||
FROM golang:1.10.1-alpine3.7
|
||||
FROM golang:1.11.4-alpine3.7
|
||||
|
||||
LABEL maintainer="Minio Inc <dev@minio.io>"
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
|
||||
RUN \
|
||||
apk add --no-cache git && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
|
||||
|
||||
FROM alpine:3.7
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
EXPOSE 9000
|
||||
|
||||
COPY --from=0 /go/bin/minio /usr/bin/minio
|
||||
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' && \
|
||||
apk add --no-cache --virtual .build-deps git && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
|
||||
rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ checks:
|
||||
@(env bash $(PWD)/buildscripts/checkgopath.sh)
|
||||
|
||||
getdeps:
|
||||
@echo "Installing golint" && go get -u github.com/golang/lint/golint
|
||||
@echo "Installing golint" && go get -u golang.org/x/lint/golint
|
||||
@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
|
||||
@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
|
||||
@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
|
||||
@@ -23,8 +23,8 @@ verifiers: getdeps vet fmt lint cyclo deadcode spelling
|
||||
|
||||
vet:
|
||||
@echo "Running $@"
|
||||
@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
|
||||
@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
|
||||
@go tool vet cmd
|
||||
@go tool vet pkg
|
||||
|
||||
fmt:
|
||||
@echo "Running $@"
|
||||
@@ -60,7 +60,7 @@ spelling:
|
||||
check: test
|
||||
test: verifiers build
|
||||
@echo "Running unit tests"
|
||||
@go test $(GOFLAGS) -tags kqueue ./...
|
||||
@CGO_ENABLED=0 go test -tags kqueue ./...
|
||||
|
||||
verify: build
|
||||
@echo "Verifying build"
|
||||
@@ -73,7 +73,7 @@ coverage: build
|
||||
# Builds minio locally.
|
||||
build: checks
|
||||
@echo "Building minio binary to './minio'"
|
||||
@CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
|
||||
@GOFLAGS="" CGO_ENABLED=0 go build -tags kqueue --ldflags $(BUILD_LDFLAGS) -o $(PWD)/minio
|
||||
|
||||
docker: build
|
||||
@docker build -t $(TAG) . -f Dockerfile.dev
|
||||
|
||||
@@ -17,7 +17,7 @@ docker run -p 9000:9000 minio/minio server /data
|
||||
docker pull minio/minio:edge
|
||||
docker run -p 9000:9000 minio/minio:edge server /data
|
||||
```
|
||||
Please visit Minio Docker quickstart guide for more [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
|
||||
Note: Docker will not display the autogenerated keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use autogenerated keys with containers. Please visit Minio Docker quickstart guide for more information [here](https://docs.minio.io/docs/minio-docker-quickstart-guide)
|
||||
|
||||
## macOS
|
||||
### Homebrew
|
||||
|
||||
@@ -1,55 +0,0 @@
|
||||
# version format
|
||||
version: "{build}"
|
||||
|
||||
# Operating system (build VM template)
|
||||
os: Windows Server 2012 R2
|
||||
|
||||
# Platform.
|
||||
platform: x64
|
||||
|
||||
clone_folder: c:\gopath\src\github.com\minio\minio
|
||||
|
||||
# Environment variables
|
||||
environment:
|
||||
GOPATH: c:\gopath
|
||||
GOROOT: c:\go110
|
||||
|
||||
# scripts that run after cloning repository
|
||||
install:
|
||||
- set PATH=%GOPATH%\bin;%GOROOT%\bin;%PATH%
|
||||
- go version
|
||||
- go env
|
||||
- python --version
|
||||
|
||||
# To run your custom scripts instead of automatic MSBuild
|
||||
build_script:
|
||||
# Compile
|
||||
# We need to disable firewall - https://github.com/appveyor/ci/issues/1579#issuecomment-309830648
|
||||
- ps: Disable-NetFirewallRule -DisplayName 'File and Printer Sharing (SMB-Out)'
|
||||
- appveyor AddCompilationMessage "Starting Compile"
|
||||
- cd c:\gopath\src\github.com\minio\minio
|
||||
- go run buildscripts/gen-ldflags.go > temp.txt
|
||||
- set /p BUILD_LDFLAGS=<temp.txt
|
||||
- go build -ldflags="%BUILD_LDFLAGS%" -o %GOPATH%\bin\minio.exe
|
||||
- appveyor AddCompilationMessage "Compile Success"
|
||||
|
||||
# To run your custom scripts instead of automatic tests
|
||||
test_script:
|
||||
# Unit tests
|
||||
- ps: Add-AppveyorTest "Unit Tests" -Outcome Running
|
||||
- mkdir build\coverage
|
||||
- for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
|
||||
- go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
|
||||
- ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
|
||||
|
||||
after_test:
|
||||
- go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
|
||||
- ps: Push-AppveyorArtifact build\coverage\coverage.txt
|
||||
- ps: Push-AppveyorArtifact build\coverage\coverage.html
|
||||
# Upload coverage report.
|
||||
- "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
|
||||
- pip install codecov
|
||||
- codecov -X gcov -f "build\coverage\coverage.txt"
|
||||
|
||||
# to disable deployment
|
||||
deploy: off
|
||||
@@ -26,6 +26,8 @@ jest.mock("../../web", () => ({
|
||||
.fn(() => true)
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(false)
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(true)
|
||||
.mockReturnValueOnce(false),
|
||||
ListObjects: jest.fn(({ bucketName }) => {
|
||||
if (bucketName === "test-deny") {
|
||||
@@ -405,7 +407,7 @@ describe("Objects actions", () => {
|
||||
store.dispatch(actionsObjects.downloadObject("obj1"))
|
||||
const url = `${
|
||||
window.location.origin
|
||||
}${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=''`
|
||||
}${minioBrowserPrefix}/download/bk1/${encodeURI("pre1/obj1")}?token=`
|
||||
expect(setLocation).toHaveBeenCalledWith(url)
|
||||
})
|
||||
|
||||
|
||||
@@ -16,11 +16,7 @@
|
||||
|
||||
import web from "../web"
|
||||
import history from "../history"
|
||||
import {
|
||||
sortObjectsByName,
|
||||
sortObjectsBySize,
|
||||
sortObjectsByDate
|
||||
} from "../utils"
|
||||
import { sortObjectsByName, sortObjectsBySize, sortObjectsByDate } from "../utils"
|
||||
import { getCurrentBucket } from "../buckets/selectors"
|
||||
import { getCurrentPrefix, getCheckedList } from "./selectors"
|
||||
import * as alertActions from "../alert/actions"
|
||||
@@ -60,10 +56,7 @@ export const appendList = (objects, marker, isTruncated) => ({
|
||||
|
||||
export const fetchObjects = append => {
|
||||
return function(dispatch, getState) {
|
||||
const {
|
||||
buckets: { currentBucket },
|
||||
objects: { currentPrefix, marker }
|
||||
} = getState()
|
||||
const {buckets: {currentBucket}, objects: {currentPrefix, marker}} = getState()
|
||||
if (currentBucket) {
|
||||
return web
|
||||
.ListObjects({
|
||||
@@ -110,7 +103,7 @@ export const fetchObjects = append => {
|
||||
|
||||
export const sortObjects = sortBy => {
|
||||
return function(dispatch, getState) {
|
||||
const { objects } = getState()
|
||||
const {objects} = getState()
|
||||
const sortOrder = objects.sortBy == sortBy ? !objects.sortOrder : true
|
||||
dispatch(setSortBy(sortBy))
|
||||
dispatch(setSortOrder(sortOrder))
|
||||
@@ -210,30 +203,39 @@ export const shareObject = (object, days, hours, minutes) => {
|
||||
const currentPrefix = getCurrentPrefix(getState())
|
||||
const objectName = `${currentPrefix}${object}`
|
||||
const expiry = days * 24 * 60 * 60 + hours * 60 * 60 + minutes * 60
|
||||
return web
|
||||
.PresignedGet({
|
||||
host: location.host,
|
||||
bucket: currentBucket,
|
||||
object: objectName,
|
||||
expiry
|
||||
})
|
||||
.then(obj => {
|
||||
dispatch(showShareObject(object, obj.url))
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "success",
|
||||
message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
|
||||
})
|
||||
)
|
||||
})
|
||||
.catch(err => {
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
})
|
||||
)
|
||||
})
|
||||
if (web.LoggedIn()) {
|
||||
return web
|
||||
.PresignedGet({
|
||||
host: location.host,
|
||||
bucket: currentBucket,
|
||||
object: objectName
|
||||
})
|
||||
.then(obj => {
|
||||
dispatch(showShareObject(object, obj.url))
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "success",
|
||||
message: `Object shared. Expires in ${days} days ${hours} hours ${minutes} minutes`
|
||||
})
|
||||
)
|
||||
})
|
||||
.catch(err => {
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
})
|
||||
)
|
||||
})
|
||||
} else {
|
||||
dispatch(showShareObject(object, `${location.host}` + '/' + `${currentBucket}` + '/' + encodeURI(objectName)))
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "success",
|
||||
message: `Object shared.`
|
||||
})
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,7 +281,7 @@ export const downloadObject = object => {
|
||||
} else {
|
||||
const url = `${
|
||||
window.location.origin
|
||||
}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=''`
|
||||
}${minioBrowserPrefix}/download/${currentBucket}/${encObjectName}?token=`
|
||||
window.location = url
|
||||
}
|
||||
}
|
||||
@@ -308,7 +310,7 @@ export const downloadCheckedObjects = () => {
|
||||
objects: getCheckedList(state)
|
||||
}
|
||||
if (!web.LoggedIn()) {
|
||||
const requestUrl = location.origin + "/minio/zip?token=''"
|
||||
const requestUrl = location.origin + "/minio/zip?token="
|
||||
downloadZip(requestUrl, req, dispatch)
|
||||
} else {
|
||||
return web
|
||||
@@ -319,14 +321,13 @@ export const downloadCheckedObjects = () => {
|
||||
}${minioBrowserPrefix}/zip?token=${res.token}`
|
||||
downloadZip(requestUrl, req, dispatch)
|
||||
})
|
||||
.catch(err =>
|
||||
dispatch(
|
||||
alertActions.set({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
})
|
||||
)
|
||||
.catch(err => dispatch(
|
||||
alertActions.set({
|
||||
type: "danger",
|
||||
message: err.message
|
||||
})
|
||||
)
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -349,8 +350,7 @@ const downloadZip = (url, req, dispatch) => {
|
||||
var separator = req.prefix.length > 1 ? "-" : ""
|
||||
|
||||
anchor.href = blobUrl
|
||||
anchor.download =
|
||||
req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
|
||||
anchor.download = req.bucketName + separator + req.prefix.slice(0, -1) + ".zip"
|
||||
|
||||
anchor.click()
|
||||
window.URL.revokeObjectURL(blobUrl)
|
||||
|
||||
+10
-9
@@ -13,8 +13,7 @@
|
||||
"setupTestFrameworkScriptFile": "./app/js/jest/setup.js",
|
||||
"testURL": "https://localhost:8080",
|
||||
"moduleNameMapper": {
|
||||
"\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$":
|
||||
"<rootDir>/app/js/jest/__mocks__/fileMock.js",
|
||||
"\\.(jpg|jpeg|png|gif|eot|otf|webp|svg|ttf|woff|woff2|mp4|webm|wav|mp3|m4a|aac|oga)$": "<rootDir>/app/js/jest/__mocks__/fileMock.js",
|
||||
"\\.(css|scss)$": "identity-obj-proxy"
|
||||
}
|
||||
},
|
||||
@@ -40,29 +39,31 @@
|
||||
"babel-preset-es2015": "^6.14.0",
|
||||
"babel-preset-react": "^6.11.1",
|
||||
"babel-register": "^6.14.0",
|
||||
"copy-webpack-plugin": "^0.3.3",
|
||||
"copy-webpack-plugin": "^4.6.0",
|
||||
"css-loader": "^0.23.1",
|
||||
"enzyme": "^3.3.0",
|
||||
"enzyme-adapter-react-16": "^1.1.1",
|
||||
"esformatter": "^0.10.0",
|
||||
"esformatter-jsx": "^7.4.1",
|
||||
"esformatter-jsx-ignore": "^1.0.6",
|
||||
"html-webpack-plugin": "^2.30.1",
|
||||
"html-webpack-plugin": "^3.2.0",
|
||||
"jest": "^22.1.4",
|
||||
"jest-enzyme": "^4.0.2",
|
||||
"json-loader": "^0.5.4",
|
||||
"less": "^2.7.1",
|
||||
"less-loader": "^2.2.3",
|
||||
"purifycss-webpack-plugin": "^2.0.3",
|
||||
"less": "^3.9.0",
|
||||
"less-loader": "^4.1.0",
|
||||
"purgecss-webpack-plugin": "^1.4.0",
|
||||
"style-loader": "^0.13.1",
|
||||
"url-loader": "^0.5.7",
|
||||
"webpack-dev-server": "^2.11.1"
|
||||
"webpack-cli": "^3.2.0",
|
||||
"webpack-dev-server": "^3.1.14"
|
||||
},
|
||||
"dependencies": {
|
||||
"bootstrap": "^3.3.6",
|
||||
"classnames": "^2.2.3",
|
||||
"expect": "^1.20.2",
|
||||
"font-awesome": "^4.7.0",
|
||||
"glob-all": "^3.1.0",
|
||||
"history": "^4.7.2",
|
||||
"humanize": "0.0.9",
|
||||
"identity-obj-proxy": "^3.0.0",
|
||||
@@ -89,6 +90,6 @@
|
||||
"reselect": "^3.0.1",
|
||||
"superagent": "^3.8.2",
|
||||
"superagent-es6-promise": "^1.0.0",
|
||||
"webpack": "^3.10.0"
|
||||
"webpack": "^4.28.3"
|
||||
}
|
||||
}
|
||||
|
||||
+42
-45
File diff suppressed because one or more lines are too long
@@ -16,11 +16,13 @@
|
||||
|
||||
var webpack = require('webpack')
|
||||
var path = require('path')
|
||||
var glob = require('glob-all')
|
||||
var CopyWebpackPlugin = require('copy-webpack-plugin')
|
||||
var purify = require("purifycss-webpack-plugin")
|
||||
var PurgecssPlugin = require('purgecss-webpack-plugin')
|
||||
|
||||
var exports = {
|
||||
context: __dirname,
|
||||
mode: 'development',
|
||||
entry: [
|
||||
path.resolve(__dirname, 'app/index.js')
|
||||
],
|
||||
@@ -99,12 +101,11 @@ var exports = {
|
||||
{from: 'app/index.html'}
|
||||
]),
|
||||
new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
|
||||
new purify({
|
||||
basePath: __dirname,
|
||||
paths: [
|
||||
"app/index.html",
|
||||
"app/js/*.js"
|
||||
]
|
||||
new PurgecssPlugin({
|
||||
paths: glob.sync([
|
||||
path.join(__dirname, 'app/index.html'),
|
||||
path.join(__dirname, 'app/js/*.js')
|
||||
])
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,11 +16,13 @@
|
||||
|
||||
var webpack = require('webpack')
|
||||
var path = require('path')
|
||||
var glob = require('glob-all')
|
||||
var CopyWebpackPlugin = require('copy-webpack-plugin')
|
||||
var purify = require("purifycss-webpack-plugin")
|
||||
var PurgecssPlugin = require('purgecss-webpack-plugin')
|
||||
|
||||
var exports = {
|
||||
context: __dirname,
|
||||
mode: 'production',
|
||||
entry: [
|
||||
path.resolve(__dirname, 'app/index.js')
|
||||
],
|
||||
@@ -74,16 +76,12 @@ var exports = {
|
||||
{from: 'app/img/logo.svg'},
|
||||
{from: 'app/index.html'}
|
||||
]),
|
||||
new webpack.DefinePlugin({
|
||||
'process.env.NODE_ENV': '"production"'
|
||||
}),
|
||||
new webpack.ContextReplacementPlugin(/moment[\\\/]locale$/, /^\.\/(en)$/),
|
||||
new purify({
|
||||
basePath: __dirname,
|
||||
paths: [
|
||||
"app/index.html",
|
||||
"app/js/*.js"
|
||||
]
|
||||
new PurgecssPlugin({
|
||||
paths: glob.sync([
|
||||
path.join(__dirname, 'app/index.html'),
|
||||
path.join(__dirname, 'app/js/*.js')
|
||||
])
|
||||
})
|
||||
]
|
||||
}
|
||||
|
||||
+2532
-1059
File diff suppressed because it is too large
Load Diff
@@ -4,7 +4,7 @@ set -e
|
||||
echo "" > coverage.txt
|
||||
|
||||
for d in $(go list ./... | grep -v browser); do
|
||||
go test -coverprofile=profile.out -covermode=atomic "$d"
|
||||
CGO_ENABLED=0 go test -v -coverprofile=profile.out -covermode=atomic "$d"
|
||||
if [ -f profile.out ]; then
|
||||
cat profile.out >> coverage.txt
|
||||
rm profile.out
|
||||
|
||||
@@ -68,6 +68,36 @@ function start_minio_erasure_sets()
|
||||
echo "$minio_pid"
|
||||
}
|
||||
|
||||
function start_minio_dist_erasure_sets_ipv6()
|
||||
{
|
||||
declare -a minio_pids
|
||||
export MINIO_ACCESS_KEY=$ACCESS_KEY
|
||||
export MINIO_SECRET_KEY=$SECRET_KEY
|
||||
"${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9000.log" 2>&1 &
|
||||
minio_pids[0]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9001.log" 2>&1 &
|
||||
minio_pids[1]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9002" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9002.log" 2>&1 &
|
||||
minio_pids[2]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9003" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9003.log" 2>&1 &
|
||||
minio_pids[3]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9004" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9004.log" 2>&1 &
|
||||
minio_pids[4]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9005" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9005.log" 2>&1 &
|
||||
minio_pids[5]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9006" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9006.log" 2>&1 &
|
||||
minio_pids[6]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9007" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9007.log" 2>&1 &
|
||||
minio_pids[7]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9008" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9008.log" 2>&1 &
|
||||
minio_pids[8]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9009" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9009.log" 2>&1 &
|
||||
minio_pids[9]=$!
|
||||
|
||||
sleep 35
|
||||
echo "${minio_pids[@]}"
|
||||
}
|
||||
|
||||
function start_minio_dist_erasure_sets()
|
||||
{
|
||||
declare -a minio_pids
|
||||
@@ -161,6 +191,34 @@ function run_test_erasure_sets() {
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function run_test_dist_erasure_sets_ipv6()
|
||||
{
|
||||
minio_pids=( $(start_minio_dist_erasure_sets_ipv6) )
|
||||
|
||||
export SERVER_ENDPOINT="[::1]:9000"
|
||||
|
||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||
rv=$?
|
||||
|
||||
for pid in "${minio_pids[@]}"; do
|
||||
kill "$pid"
|
||||
done
|
||||
sleep 3
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
for i in $(seq 0 9); do
|
||||
echo "server$i log:"
|
||||
cat "$WORK_DIR/dist-minio-v6-900$i.log"
|
||||
done
|
||||
fi
|
||||
|
||||
for i in $(seq 0 9); do
|
||||
rm -f "$WORK_DIR/dist-minio-v6-900$i.log"
|
||||
done
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function run_test_dist_erasure_sets()
|
||||
{
|
||||
minio_pids=( $(start_minio_dist_erasure_sets) )
|
||||
@@ -237,6 +295,7 @@ function run_test_gateway_s3()
|
||||
{
|
||||
minio_pid="$(start_minio_gateway_s3)"
|
||||
|
||||
export SERVER_ENDPOINT="127.0.0.1:9000"
|
||||
export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||
@@ -279,6 +338,7 @@ function __init__()
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
|
||||
chmod a+x "$FUNCTIONAL_TESTS"
|
||||
}
|
||||
|
||||
@@ -319,6 +379,13 @@ function main()
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Distributed Erasure setup as sets with ipv6"
|
||||
if ! run_test_dist_erasure_sets_ipv6; then
|
||||
echo "FAILED"
|
||||
rm -fr "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Gateway S3 setup"
|
||||
if ! run_test_gateway_s3; then
|
||||
echo "FAILED"
|
||||
|
||||
+13
-8
@@ -21,6 +21,7 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
@@ -56,26 +57,28 @@ type accessControlPolicy struct {
|
||||
func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketACL")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Allow getBucketACL if policy action is set, since this is a dummy call
|
||||
// we are simply re-purposing the bucketPolicyAction.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -89,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
Permission: "FULL_CONTROL",
|
||||
})
|
||||
if err := xml.NewEncoder(w).Encode(acl); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -103,27 +106,29 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetObjectACL")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Allow getObjectACL if policy action is set, since this is a dummy call
|
||||
// we are simply re-purposing the bucketPolicyAction.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Before proceeding validate if object exists.
|
||||
_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -137,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
|
||||
Permission: "FULL_CONTROL",
|
||||
})
|
||||
if err := xml.NewEncoder(w).Encode(acl); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+573
-104
@@ -35,7 +35,9 @@ import (
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
"github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
"github.com/tidwall/gjson"
|
||||
@@ -43,7 +45,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
maxConfigJSONSize = 256 * 1024 // 256KiB
|
||||
maxEConfigJSONSize = 262272
|
||||
)
|
||||
|
||||
// Type-safe query params.
|
||||
@@ -52,9 +54,10 @@ type mgmtQueryKey string
|
||||
// Only valid query params for mgmt admin APIs.
|
||||
const (
|
||||
mgmtBucket mgmtQueryKey = "bucket"
|
||||
mgmtPrefix mgmtQueryKey = "prefix"
|
||||
mgmtClientToken mgmtQueryKey = "clientToken"
|
||||
mgmtForceStart mgmtQueryKey = "forceStart"
|
||||
mgmtPrefix = "prefix"
|
||||
mgmtClientToken = "clientToken"
|
||||
mgmtForceStart = "forceStart"
|
||||
mgmtForceStop = "forceStop"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -69,7 +72,9 @@ var (
|
||||
// -----------
|
||||
// Returns Administration API version
|
||||
func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request) {
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
ctx := newContext(r, w, "Version")
|
||||
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -77,8 +82,7 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
jsonBytes, err := json.Marshal(adminAPIVersionInfo)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, ErrInternalError, r.URL)
|
||||
logger.LogIf(context.Background(), err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -89,7 +93,9 @@ func (a adminAPIHandlers) VersionHandler(w http.ResponseWriter, r *http.Request)
|
||||
// ----------
|
||||
// Returns server version and uptime.
|
||||
func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
ctx := newContext(r, w, "ServiceStatus")
|
||||
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -105,8 +111,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
|
||||
// of read-quorum availability.
|
||||
uptime, err := getPeerUptimes(globalAdminPeers)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
|
||||
logger.LogIf(context.Background(), err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -119,8 +124,7 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(serverStatus)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, ErrInternalError, r.URL)
|
||||
logger.LogIf(context.Background(), err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Reply with storage information (across nodes in a
|
||||
@@ -134,7 +138,9 @@ func (a adminAPIHandlers) ServiceStatusHandler(w http.ResponseWriter, r *http.Re
|
||||
// Restarts/Stops minio server gracefully. In a distributed setup,
|
||||
// restarts all the servers in the cluster.
|
||||
func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *http.Request) {
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
ctx := newContext(r, w, "ServiceStopNRestart")
|
||||
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -143,7 +149,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
|
||||
var sa madmin.ServiceAction
|
||||
err := json.NewDecoder(r.Body).Decode(&sa)
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, ErrRequestBodyParse, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -156,7 +162,7 @@ func (a adminAPIHandlers) ServiceStopNRestartHandler(w http.ResponseWriter, r *h
|
||||
serviceSig = serviceStop
|
||||
default:
|
||||
writeErrorResponseJSON(w, ErrMalformedPOSTRequest, r.URL)
|
||||
logger.LogIf(context.Background(), errors.New("Invalid service action received"))
|
||||
logger.LogIf(ctx, errors.New("Invalid service action received"))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -225,10 +231,12 @@ type ServerInfo struct {
|
||||
// ----------
|
||||
// Get server information
|
||||
func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ServerInfo")
|
||||
|
||||
// Authenticate request
|
||||
|
||||
// Setting the region as empty so as the mc server info command is irrespective to the region.
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -253,7 +261,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
serverInfoData, err := peer.cmdRunner.ServerInfo()
|
||||
if err != nil {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", peer.addr)
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, err)
|
||||
reply[idx].Error = err.Error()
|
||||
return
|
||||
@@ -268,8 +276,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(reply)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, ErrInternalError, r.URL)
|
||||
logger.LogIf(context.Background(), err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -278,6 +285,57 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
}
|
||||
|
||||
// ServerDrivesPerfInfo holds informantion about address, performance
|
||||
// of all drives on one server. It also reports any errors if encountered
|
||||
// while trying to reach this server.
|
||||
type ServerDrivesPerfInfo struct {
|
||||
Addr string `json:"addr"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Perf []disk.Performance `json:"perf"`
|
||||
}
|
||||
|
||||
// PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
|
||||
// ----------
|
||||
// Get all performance information based on input type
|
||||
// Supported types = drive
|
||||
func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PerfInfo")
|
||||
|
||||
// Authenticate request
|
||||
// Setting the region as empty so as the mc server info command is irrespective to the region.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
perfType := vars["perfType"]
|
||||
|
||||
if perfType == "drive" {
|
||||
// Get drive performance details from local server's drive(s)
|
||||
dp := localEndpointsPerf(globalEndpoints)
|
||||
|
||||
// Notify all other Minio peers to report drive performance numbers
|
||||
dps := globalNotificationSys.DrivePerfInfo()
|
||||
dps = append(dps, dp)
|
||||
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(dps)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Reply with performance information (across nodes in a
|
||||
// distributed setup) as json.
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
} else {
|
||||
writeErrorResponseJSON(w, ErrNotImplemented, r.URL)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// StartProfilingResult contains the status of the starting
|
||||
// profiling action in a given server
|
||||
type StartProfilingResult struct {
|
||||
@@ -290,7 +348,9 @@ type StartProfilingResult struct {
|
||||
// ----------
|
||||
// Enable server profiling
|
||||
func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
ctx := newContext(r, w, "StartProfiling")
|
||||
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -349,7 +409,9 @@ func (f dummyFileInfo) Sys() interface{} { return f.sys }
|
||||
// ----------
|
||||
// Download profiling information of all nodes in a zip format
|
||||
func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
ctx := newContext(r, w, "DownloadProfiling")
|
||||
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -366,7 +428,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
|
||||
// Get profiling data from a node
|
||||
data, err := peer.cmdRunner.DownloadProfilingData()
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to download profiling data from node `%s`, reason: %s", peer.addr, err.Error()))
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -401,7 +463,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
// extractHealInitParams - Validates params for heal init API.
|
||||
func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
|
||||
hs madmin.HealOpts, clientToken string, forceStart bool,
|
||||
hs madmin.HealOpts, clientToken string, forceStart bool, forceStop bool,
|
||||
err APIErrorCode) {
|
||||
|
||||
vars := mux.Vars(r)
|
||||
@@ -432,7 +494,9 @@ func extractHealInitParams(r *http.Request) (bucket, objPrefix string,
|
||||
if _, ok := qParms[string(mgmtForceStart)]; ok {
|
||||
forceStart = true
|
||||
}
|
||||
|
||||
if _, ok := qParms[string(mgmtForceStop)]; ok {
|
||||
forceStop = true
|
||||
}
|
||||
// ignore body if clientToken is provided
|
||||
if clientToken == "" {
|
||||
jerr := json.NewDecoder(r.Body).Decode(&hs)
|
||||
@@ -471,7 +535,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -483,7 +547,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
bucket, objPrefix, hs, clientToken, forceStart, apiErr := extractHealInitParams(r)
|
||||
bucket, objPrefix, hs, clientToken, forceStart, forceStop, apiErr := extractHealInitParams(r)
|
||||
if apiErr != ErrNone {
|
||||
writeErrorResponseJSON(w, apiErr, r.URL)
|
||||
return
|
||||
@@ -517,13 +581,35 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Write([]byte("\n\r"))
|
||||
w.(http.Flusher).Flush()
|
||||
case hr := <-respCh:
|
||||
switch {
|
||||
case hr.errCode == ErrNone:
|
||||
writeSuccessResponseJSON(w, hr.respBytes)
|
||||
case hr.errBody == "":
|
||||
writeErrorResponseJSON(w, hr.errCode, r.URL)
|
||||
switch hr.errCode {
|
||||
case ErrNone:
|
||||
if started {
|
||||
w.Write(hr.respBytes)
|
||||
w.(http.Flusher).Flush()
|
||||
} else {
|
||||
writeSuccessResponseJSON(w, hr.respBytes)
|
||||
}
|
||||
default:
|
||||
writeCustomErrorResponseJSON(w, hr.errCode, hr.errBody, r.URL)
|
||||
apiError := getAPIError(hr.errCode)
|
||||
var errorRespJSON []byte
|
||||
if hr.errBody == "" {
|
||||
errorRespJSON = encodeResponseJSON(getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey)))
|
||||
} else {
|
||||
errorRespJSON = encodeResponseJSON(APIErrorResponse{
|
||||
Code: apiError.Code,
|
||||
Message: hr.errBody,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
HostID: "3L137",
|
||||
})
|
||||
}
|
||||
if !started {
|
||||
setCommonHeaders(w)
|
||||
w.Header().Set("Content-Type", string(mimeJSON))
|
||||
w.WriteHeader(apiError.HTTPStatusCode)
|
||||
}
|
||||
w.Write(errorRespJSON)
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
break forLoop
|
||||
}
|
||||
@@ -534,34 +620,60 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
info := objLayer.StorageInfo(ctx)
|
||||
numDisks := info.Backend.OfflineDisks + info.Backend.OnlineDisks
|
||||
|
||||
if clientToken == "" {
|
||||
// Not a status request
|
||||
nh := newHealSequence(bucket, objPrefix, handlers.GetSourceIP(r),
|
||||
numDisks, hs, forceStart)
|
||||
healPath := pathJoin(bucket, objPrefix)
|
||||
if clientToken == "" && !forceStart && !forceStop {
|
||||
nh, exists := globalAllHealState.getHealSequence(healPath)
|
||||
if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
ClientToken: nh.clientToken,
|
||||
ClientAddress: nh.clientAddress,
|
||||
StartTime: nh.startTime,
|
||||
})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Client token not specified but a heal sequence exists on a path,
|
||||
// Send the token back to client.
|
||||
writeSuccessResponseJSON(w, b)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
respCh := make(chan healResp)
|
||||
go func() {
|
||||
respBytes, errCode, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
|
||||
hr := healResp{respBytes, errCode, errMsg}
|
||||
respCh <- hr
|
||||
}()
|
||||
|
||||
// Due to the force-starting functionality, the Launch
|
||||
// call above can take a long time - to keep the
|
||||
// connection alive, we start sending whitespace
|
||||
keepConnLive(w, respCh)
|
||||
} else {
|
||||
if clientToken != "" && !forceStart && !forceStop {
|
||||
// Since clientToken is given, fetch heal status from running
|
||||
// heal sequence.
|
||||
path := bucket + "/" + objPrefix
|
||||
respBytes, errCode := globalAllHealState.PopHealStatusJSON(
|
||||
path, clientToken)
|
||||
healPath, clientToken)
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponseJSON(w, errCode, r.URL)
|
||||
} else {
|
||||
writeSuccessResponseJSON(w, respBytes)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
respCh := make(chan healResp)
|
||||
switch {
|
||||
case forceStop:
|
||||
go func() {
|
||||
respBytes, errCode := globalAllHealState.stopHealSequence(healPath)
|
||||
hr := healResp{respBytes: respBytes, errCode: errCode}
|
||||
respCh <- hr
|
||||
}()
|
||||
case clientToken == "":
|
||||
nh := newHealSequence(bucket, objPrefix, handlers.GetSourceIP(r), numDisks, hs, forceStart)
|
||||
go func() {
|
||||
respBytes, errCode, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
|
||||
hr := healResp{respBytes, errCode, errMsg}
|
||||
respCh <- hr
|
||||
}()
|
||||
}
|
||||
|
||||
// Due to the force-starting functionality, the Launch
|
||||
// call above can take a long time - to keep the
|
||||
// connection alive, we start sending whitespace
|
||||
keepConnLive(w, respCh)
|
||||
}
|
||||
|
||||
// GetConfigHandler - GET /minio/admin/v1/config
|
||||
@@ -577,7 +689,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -585,22 +697,20 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
|
||||
config, err := readServerConfig(ctx, objectAPI)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := json.MarshalIndent(config, "", "\t")
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := config.GetCredential().SecretKey
|
||||
econfigData, err := madmin.EncryptServerConfigData(password, configData)
|
||||
econfigData, err := madmin.EncryptData(password, configData)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -639,7 +749,7 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -654,14 +764,13 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
|
||||
config, err := readServerConfig(ctx, objectAPI)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
configData, err := json.Marshal(config)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -681,10 +790,9 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
password := config.GetCredential().SecretKey
|
||||
econfigData, err := madmin.EncryptServerConfigData(password, []byte(newConfigStr))
|
||||
econfigData, err := madmin.EncryptData(password, []byte(newConfigStr))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -693,12 +801,384 @@ func (a adminAPIHandlers) GetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// toAdminAPIErrCode - converts errXLWriteQuorum error to admin API
|
||||
// specific error.
|
||||
func toAdminAPIErrCode(err error) APIErrorCode {
|
||||
func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
|
||||
switch err {
|
||||
case errXLWriteQuorum:
|
||||
return ErrAdminConfigNoQuorum
|
||||
default:
|
||||
return toAPIErrorCode(err)
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveUser - DELETE /minio/admin/v1/remove-user?accessKey=<access_key>
|
||||
func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RemoveUser")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Deny if WORM is enabled
|
||||
if globalWORMEnabled {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
if err := globalIAMSys.DeleteUser(accessKey); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
}
|
||||
}
|
||||
|
||||
// ListUsers - GET /minio/admin/v1/list-users
|
||||
func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListUsers")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
allCredentials, err := globalIAMSys.ListUsers()
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
data, err := json.Marshal(allCredentials)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := globalServerConfig.GetCredential().SecretKey
|
||||
econfigData, err := madmin.EncryptData(password, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, econfigData)
|
||||
}
|
||||
|
||||
// SetUserStatus - PUT /minio/admin/v1/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
|
||||
func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "SetUserStatus")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Deny if WORM is enabled
|
||||
if globalWORMEnabled {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
status := vars["status"]
|
||||
|
||||
// Custom IAM policies not allowed for admin user.
|
||||
if accessKey == globalServerConfig.GetCredential().AccessKey {
|
||||
writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalIAMSys.SetUserStatus(accessKey, madmin.AccountStatus(status)); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddUser - PUT /minio/admin/v1/add-user?accessKey=<access_key>
|
||||
func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AddUser")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Deny if WORM is enabled
|
||||
if globalWORMEnabled {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
|
||||
// Custom IAM policies not allowed for admin user.
|
||||
if accessKey == globalServerConfig.GetCredential().AccessKey {
|
||||
writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
|
||||
// More than maxConfigSize bytes were available
|
||||
writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := globalServerConfig.GetCredential().SecretKey
|
||||
configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var uinfo madmin.UserInfo
|
||||
if err = json.Unmarshal(configBytes, &uinfo); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = globalIAMSys.SetUser(accessKey, uinfo); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
|
||||
func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListCannedPolicies")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
policies, err := globalIAMSys.ListCannedPolicies()
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = json.NewEncoder(w).Encode(policies); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
w.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// RemoveCannedPolicy - DELETE /minio/admin/v1/remove-canned-policy?name=<policy_name>
|
||||
func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "RemoveCannedPolicy")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
policyName := vars["name"]
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Deny if WORM is enabled
|
||||
if globalWORMEnabled {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalIAMSys.DeleteCannedPolicy(policyName); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddCannedPolicy - PUT /minio/admin/v1/add-canned-policy?name=<policy_name>
|
||||
func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AddCannedPolicy")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
policyName := vars["name"]
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Deny if WORM is enabled
|
||||
if globalWORMEnabled {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is missing.
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponseJSON(w, ErrMissingContentLength, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
if r.ContentLength > maxBucketPolicySize {
|
||||
writeErrorResponseJSON(w, ErrEntityTooLarge, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, ErrMalformedPolicy, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Version in policy must not be empty
|
||||
if iamPolicy.Version == "" {
|
||||
writeErrorResponseJSON(w, ErrMalformedPolicy, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = globalIAMSys.SetCannedPolicy(policyName, *iamPolicy); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetUserPolicy - PUT /minio/admin/v1/set-user-policy?accessKey=<access_key>&name=<policy_name>
|
||||
func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "SetUserPolicy")
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
policyName := vars["name"]
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Deny if WORM is enabled
|
||||
if globalWORMEnabled {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Custom IAM policies not allowed for admin user.
|
||||
if accessKey == globalServerConfig.GetCredential().AccessKey {
|
||||
writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -714,7 +1194,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -726,22 +1206,14 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
// Read configuration bytes from request body.
|
||||
configBuf := make([]byte, maxConfigJSONSize+1)
|
||||
n, err := io.ReadFull(r.Body, configBuf)
|
||||
if err == nil {
|
||||
if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
|
||||
// More than maxConfigSize bytes were available
|
||||
writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
|
||||
return
|
||||
}
|
||||
if err != io.ErrUnexpectedEOF {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := globalServerConfig.GetCredential().SecretKey
|
||||
configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
|
||||
configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
|
||||
@@ -757,8 +1229,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
|
||||
var config serverConfig
|
||||
err = json.Unmarshal(configBytes, &config)
|
||||
if err != nil {
|
||||
if err = json.Unmarshal(configBytes, &config); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
|
||||
return
|
||||
@@ -780,8 +1251,13 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
return
|
||||
}
|
||||
|
||||
if err = config.TestNotificationTargets(); err != nil {
|
||||
writeCustomErrorResponseJSON(w, ErrAdminConfigBadJSON, err.Error(), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -823,7 +1299,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
@@ -832,15 +1308,14 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
// Load config
|
||||
configStruct, err := readServerConfig(ctx, objectAPI)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Convert config to json bytes
|
||||
configBytes, err := json.Marshal(configStruct)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -860,7 +1335,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
|
||||
return
|
||||
}
|
||||
elem, dErr := madmin.DecryptServerConfigData(password, bytes.NewBuffer([]byte(encryptedElem)))
|
||||
elem, dErr := madmin.DecryptData(password, bytes.NewBuffer([]byte(encryptedElem)))
|
||||
if dErr != nil {
|
||||
logger.LogIf(ctx, dErr)
|
||||
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
|
||||
@@ -912,7 +1387,7 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
if err = saveServerConfig(ctx, objectAPI, &config); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -920,11 +1395,10 @@ func (a adminAPIHandlers) SetConfigKeysHandler(w http.ResponseWriter, r *http.Re
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// UpdateCredsHandler - POST /minio/admin/v1/config/credential
|
||||
// UpdateAdminCredsHandler - POST /minio/admin/v1/config/credential
|
||||
// ----------
|
||||
// Update credentials in a minio server. In a distributed setup,
|
||||
// update all the servers in the cluster.
|
||||
func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
|
||||
// Update admin credentials in a minio server
|
||||
func (a adminAPIHandlers) UpdateAdminCredentialsHandler(w http.ResponseWriter,
|
||||
r *http.Request) {
|
||||
|
||||
ctx := newContext(r, w, "UpdateCredentialsHandler")
|
||||
@@ -938,34 +1412,26 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
|
||||
|
||||
// Avoid setting new credentials when they are already passed
|
||||
// by the environment. Deny if WORM is enabled.
|
||||
if globalIsEnvCreds {
|
||||
if globalIsEnvCreds || globalWORMEnabled {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Authenticate request
|
||||
adminAPIErr := checkAdminRequestAuthType(r, "")
|
||||
adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Read configuration bytes from request body.
|
||||
configBuf := make([]byte, maxConfigJSONSize+1)
|
||||
n, err := io.ReadFull(r.Body, configBuf)
|
||||
if err == nil {
|
||||
if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
|
||||
// More than maxConfigSize bytes were available
|
||||
writeErrorResponseJSON(w, ErrAdminConfigTooLarge, r.URL)
|
||||
return
|
||||
}
|
||||
if err != io.ErrUnexpectedEOF {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := globalServerConfig.GetCredential().SecretKey
|
||||
configBytes, err := madmin.DecryptServerConfigData(password, bytes.NewReader(configBuf[:n]))
|
||||
configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(w, ErrAdminConfigBadJSON, r.URL)
|
||||
@@ -982,7 +1448,7 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
|
||||
|
||||
creds, err := auth.CreateCredentials(req.AccessKey, req.SecretKey)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -993,8 +1459,11 @@ func (a adminAPIHandlers) UpdateCredentialsHandler(w http.ResponseWriter,
|
||||
// Update local credentials in memory.
|
||||
globalServerConfig.SetCredential(creds)
|
||||
|
||||
// Set active creds.
|
||||
globalActiveCred = creds
|
||||
|
||||
if err = saveServerConfig(ctx, objectAPI, globalServerConfig); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+199
-175
@@ -38,172 +38,194 @@ import (
|
||||
|
||||
var (
|
||||
configJSON = []byte(`{
|
||||
"version": "30",
|
||||
"credential": {
|
||||
"accessKey": "minio",
|
||||
"secretKey": "minio123"
|
||||
"version": "33",
|
||||
"credential": {
|
||||
"accessKey": "minio",
|
||||
"secretKey": "minio123"
|
||||
},
|
||||
"region": "us-east-1",
|
||||
"worm": "off",
|
||||
"storageclass": {
|
||||
"standard": "",
|
||||
"rrs": ""
|
||||
},
|
||||
"cache": {
|
||||
"drives": [],
|
||||
"expiry": 90,
|
||||
"maxuse": 80,
|
||||
"exclude": []
|
||||
},
|
||||
"kms": {
|
||||
"vault": {
|
||||
"endpoint": "",
|
||||
"auth": {
|
||||
"type": "",
|
||||
"approle": {
|
||||
"id": "",
|
||||
"secret": ""
|
||||
}
|
||||
},
|
||||
"key-id": {
|
||||
"name": "",
|
||||
"version": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"notify": {
|
||||
"amqp": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"url": "",
|
||||
"exchange": "",
|
||||
"routingKey": "",
|
||||
"exchangeType": "",
|
||||
"deliveryMode": 0,
|
||||
"mandatory": false,
|
||||
"immediate": false,
|
||||
"durable": false,
|
||||
"internal": false,
|
||||
"noWait": false,
|
||||
"autoDeleted": false
|
||||
}
|
||||
},
|
||||
"elasticsearch": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "namespace",
|
||||
"url": "",
|
||||
"index": ""
|
||||
}
|
||||
},
|
||||
"kafka": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"brokers": null,
|
||||
"topic": "",
|
||||
"tls": {
|
||||
"enable": false,
|
||||
"skipVerify": false,
|
||||
"clientAuth": 0
|
||||
},
|
||||
"sasl": {
|
||||
"enable": false,
|
||||
"username": "",
|
||||
"password": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
"mqtt": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"broker": "",
|
||||
"topic": "",
|
||||
"qos": 0,
|
||||
"clientId": "",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"reconnectInterval": 0,
|
||||
"keepAliveInterval": 0
|
||||
}
|
||||
},
|
||||
"mysql": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "namespace",
|
||||
"dsnString": "",
|
||||
"table": "",
|
||||
"host": "",
|
||||
"port": "",
|
||||
"user": "",
|
||||
"password": "",
|
||||
"database": ""
|
||||
}
|
||||
},
|
||||
"nats": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"address": "",
|
||||
"subject": "",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"token": "",
|
||||
"secure": false,
|
||||
"pingInterval": 0,
|
||||
"streaming": {
|
||||
"enable": false,
|
||||
"clusterID": "",
|
||||
"async": false,
|
||||
"maxPubAcksInflight": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"region": "",
|
||||
"worm": "off",
|
||||
"storageclass": {
|
||||
"standard": "",
|
||||
"rrs": ""
|
||||
},
|
||||
"cache": {
|
||||
"drives": [],
|
||||
"expiry": 90,
|
||||
"maxuse": 80,
|
||||
"exclude": []
|
||||
},
|
||||
"kms": {
|
||||
"vault": {
|
||||
"endpoint": "",
|
||||
"auth": {
|
||||
"type": "",
|
||||
"approle": {
|
||||
"id": "",
|
||||
"secret": ""
|
||||
}
|
||||
},
|
||||
"key-id": {
|
||||
"name": "",
|
||||
"version": 0
|
||||
}
|
||||
"nsq": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"nsqdAddress": "",
|
||||
"topic": "",
|
||||
"tls": {
|
||||
"enable": false,
|
||||
"skipVerify": false
|
||||
}
|
||||
},
|
||||
"notify": {
|
||||
"amqp": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"url": "",
|
||||
"exchange": "",
|
||||
"routingKey": "",
|
||||
"exchangeType": "",
|
||||
"deliveryMode": 0,
|
||||
"mandatory": false,
|
||||
"immediate": false,
|
||||
"durable": false,
|
||||
"internal": false,
|
||||
"noWait": false,
|
||||
"autoDeleted": false
|
||||
}
|
||||
},
|
||||
"elasticsearch": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "",
|
||||
"url": "",
|
||||
"index": ""
|
||||
}
|
||||
},
|
||||
"kafka": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"brokers": null,
|
||||
"topic": "",
|
||||
"tls" : {
|
||||
"enable" : false,
|
||||
"skipVerify" : false,
|
||||
"clientAuth" : 0
|
||||
},
|
||||
"sasl" : {
|
||||
"enable" : false,
|
||||
"username" : "",
|
||||
"password" : ""
|
||||
}
|
||||
}
|
||||
},
|
||||
"mqtt": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"broker": "",
|
||||
"topic": "",
|
||||
"qos": 0,
|
||||
"clientId": "",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"reconnectInterval": 0,
|
||||
"keepAliveInterval": 0
|
||||
}
|
||||
},
|
||||
"mysql": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "",
|
||||
"dsnString": "",
|
||||
"table": "",
|
||||
"host": "",
|
||||
"port": "",
|
||||
"user": "",
|
||||
"password": "",
|
||||
"database": ""
|
||||
}
|
||||
},
|
||||
"nats": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"address": "",
|
||||
"subject": "",
|
||||
"username": "",
|
||||
"password": "",
|
||||
"token": "",
|
||||
"secure": false,
|
||||
"pingInterval": 0,
|
||||
"streaming": {
|
||||
"enable": false,
|
||||
"clusterID": "",
|
||||
"clientID": "",
|
||||
"async": false,
|
||||
"maxPubAcksInflight": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
"postgresql": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "",
|
||||
"connectionString": "",
|
||||
"table": "",
|
||||
"host": "",
|
||||
"port": "",
|
||||
"user": "",
|
||||
"password": "",
|
||||
"database": ""
|
||||
}
|
||||
},
|
||||
"redis": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "",
|
||||
"address": "",
|
||||
"password": "",
|
||||
"key": ""
|
||||
}
|
||||
},
|
||||
"webhook": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"endpoint": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
"logger": {
|
||||
"console": {
|
||||
"enabled": true
|
||||
},
|
||||
"http": {
|
||||
"target1": {
|
||||
"enabled": false,
|
||||
"endpoint": "https://username:password@example.com/api"
|
||||
}
|
||||
}
|
||||
},
|
||||
"compress": {
|
||||
"enabled": false,
|
||||
"extensions":[".txt",".log",".csv",".json"],
|
||||
"mime-types":["text/csv","text/plain","application/json"]
|
||||
}
|
||||
}`)
|
||||
}
|
||||
},
|
||||
"postgresql": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "namespace",
|
||||
"connectionString": "",
|
||||
"table": "",
|
||||
"host": "",
|
||||
"port": "",
|
||||
"user": "",
|
||||
"password": "",
|
||||
"database": ""
|
||||
}
|
||||
},
|
||||
"redis": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"format": "namespace",
|
||||
"address": "",
|
||||
"password": "",
|
||||
"key": ""
|
||||
}
|
||||
},
|
||||
"webhook": {
|
||||
"1": {
|
||||
"enable": false,
|
||||
"endpoint": ""
|
||||
}
|
||||
}
|
||||
},
|
||||
"logger": {
|
||||
"console": {
|
||||
"enabled": true
|
||||
},
|
||||
"http": {
|
||||
"1": {
|
||||
"enabled": false,
|
||||
"endpoint": "https://username:password@example.com/api"
|
||||
}
|
||||
}
|
||||
},
|
||||
"compress": {
|
||||
"enabled": false,
|
||||
"extensions":[".txt",".log",".csv",".json"],
|
||||
"mime-types":["text/csv","text/plain","application/json"]
|
||||
},
|
||||
"openid": {
|
||||
"jwks": {
|
||||
"url": ""
|
||||
}
|
||||
},
|
||||
"policy": {
|
||||
"opa": {
|
||||
"url": "",
|
||||
"authToken": ""
|
||||
}
|
||||
}
|
||||
}
|
||||
`)
|
||||
)
|
||||
|
||||
// adminXLTestBed - encapsulates subsystems that need to be setup for
|
||||
@@ -255,7 +277,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := mux.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
registerAdminRouter(adminRouter, true)
|
||||
|
||||
return &adminXLTestBed{
|
||||
xlDirs: xlDirs,
|
||||
@@ -286,7 +308,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
|
||||
for i := 0; i < 10; i++ {
|
||||
objectName := fmt.Sprintf("%s-%d", objName, i)
|
||||
_, err = atb.objLayer.PutObject(context.Background(), bucketName, objectName,
|
||||
mustGetHashReader(t, bytes.NewReader([]byte("hello")),
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte("hello")),
|
||||
int64(len("hello")), "", ""), nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create %s - %v", objectName,
|
||||
@@ -305,7 +327,7 @@ func (atb *adminXLTestBed) GenerateHealTestData(t *testing.T) {
|
||||
}
|
||||
|
||||
_, err = atb.objLayer.PutObjectPart(context.Background(), bucketName, objName,
|
||||
uploadID, 3, mustGetHashReader(t, bytes.NewReader(
|
||||
uploadID, 3, mustGetPutObjReader(t, bytes.NewReader(
|
||||
[]byte("hello")), int64(len("hello")), "", ""), ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatalf("mp put error: %v", err)
|
||||
@@ -485,6 +507,8 @@ func getServiceCmdRequest(cmd cmdType, cred auth.Credentials, body []byte) (*htt
|
||||
|
||||
// Set body
|
||||
req.Body = ioutil.NopCloser(bytes.NewReader(body))
|
||||
req.ContentLength = int64(len(body))
|
||||
|
||||
// Set sha-sum header
|
||||
req.Header.Set("X-Amz-Content-Sha256", getSHA256Hash(body))
|
||||
|
||||
@@ -525,7 +549,7 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
|
||||
credentials := globalServerConfig.GetCredential()
|
||||
|
||||
body, err := json.Marshal(madmin.ServiceAction{
|
||||
cmd.toServiceActionValue()})
|
||||
Action: cmd.toServiceActionValue()})
|
||||
if err != nil {
|
||||
t.Fatalf("JSONify error: %v", err)
|
||||
}
|
||||
@@ -615,7 +639,7 @@ func TestServiceSetCreds(t *testing.T) {
|
||||
t.Fatalf("JSONify err: %v", err)
|
||||
}
|
||||
|
||||
ebody, err := madmin.EncryptServerConfigData(credentials.SecretKey, body)
|
||||
ebody, err := madmin.EncryptData(credentials.SecretKey, body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -718,7 +742,7 @@ func TestSetConfigHandler(t *testing.T) {
|
||||
queryVal.Set("config", "")
|
||||
|
||||
password := globalServerConfig.GetCredential().SecretKey
|
||||
econfigJSON, err := madmin.EncryptServerConfigData(password, configJSON)
|
||||
econfigJSON, err := madmin.EncryptData(password, configJSON)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -732,13 +756,13 @@ func TestSetConfigHandler(t *testing.T) {
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.router.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("Expected to succeed but failed with %d", rec.Code)
|
||||
t.Errorf("Expected to succeed but failed with %d, body: %s", rec.Code, rec.Body)
|
||||
}
|
||||
|
||||
// Check that a very large config file returns an error.
|
||||
{
|
||||
// Make a large enough config string
|
||||
invalidCfg := []byte(strings.Repeat("A", maxConfigJSONSize+1))
|
||||
invalidCfg := []byte(strings.Repeat("A", maxEConfigJSONSize+1))
|
||||
req, err := buildAdminRequest(queryVal, http.MethodPut, "/config",
|
||||
int64(len(invalidCfg)), bytes.NewReader(invalidCfg))
|
||||
if err != nil {
|
||||
@@ -768,7 +792,7 @@ func TestSetConfigHandler(t *testing.T) {
|
||||
adminTestBed.router.ServeHTTP(rec, req)
|
||||
respBody := string(rec.Body.Bytes())
|
||||
if rec.Code != http.StatusBadRequest ||
|
||||
!strings.Contains(respBody, "JSON configuration provided has objects with duplicate keys") {
|
||||
!strings.Contains(respBody, "JSON configuration provided is of incorrect format") {
|
||||
t.Errorf("Got unexpected response code or body %d - %s", rec.Code, respBody)
|
||||
}
|
||||
}
|
||||
@@ -842,12 +866,12 @@ func TestToAdminAPIErr(t *testing.T) {
|
||||
// 3. Non-admin API specific error.
|
||||
{
|
||||
err: errDiskNotFound,
|
||||
expectedAPIErr: toAPIErrorCode(errDiskNotFound),
|
||||
expectedAPIErr: toAPIErrorCode(context.Background(), errDiskNotFound),
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
actualErr := toAdminAPIErrCode(test.err)
|
||||
actualErr := toAdminAPIErrCode(context.Background(), test.err)
|
||||
if actualErr != test.expectedAPIErr {
|
||||
t.Errorf("Test %d: Expected %v but received %v",
|
||||
i+1, test.expectedAPIErr, actualErr)
|
||||
|
||||
+84
-53
@@ -20,6 +20,7 @@ import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
@@ -61,8 +62,8 @@ var (
|
||||
errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
|
||||
errHealStopSignalled = fmt.Errorf("heal stop signaled")
|
||||
|
||||
errFnHealFromAPIErr = func(err error) error {
|
||||
errCode := toAPIErrorCode(err)
|
||||
errFnHealFromAPIErr = func(ctx context.Context, err error) error {
|
||||
errCode := toAPIErrorCode(ctx, err)
|
||||
apiErr := getAPIError(errCode)
|
||||
return fmt.Errorf("Heal internal error: %s: %s",
|
||||
apiErr.Code, apiErr.Description)
|
||||
@@ -112,6 +113,32 @@ func initAllHealState(isErasureMode bool) {
|
||||
globalAllHealState = allHealState{
|
||||
healSeqMap: make(map[string]*healSequence),
|
||||
}
|
||||
|
||||
go globalAllHealState.periodicHealSeqsClean()
|
||||
}
|
||||
|
||||
func (ahs *allHealState) periodicHealSeqsClean() {
|
||||
// Launch clean-up routine to remove this heal sequence (after
|
||||
// it ends) from the global state after timeout has elapsed.
|
||||
ticker := time.NewTicker(time.Minute * 5)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
now := UTCNow()
|
||||
ahs.Lock()
|
||||
for path, h := range ahs.healSeqMap {
|
||||
if h.hasEnded() && h.endTime.Add(keepHealSeqStateDuration).Before(now) {
|
||||
delete(ahs.healSeqMap, path)
|
||||
}
|
||||
}
|
||||
ahs.Unlock()
|
||||
case <-GlobalServiceDoneCh:
|
||||
// server could be restarting - need
|
||||
// to exit immediately
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// getHealSequence - Retrieve a heal sequence by path. The second
|
||||
@@ -123,6 +150,35 @@ func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists b
|
||||
return h, exists
|
||||
}
|
||||
|
||||
func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIErrorCode) {
|
||||
var hsp madmin.HealStopSuccess
|
||||
he, exists := ahs.getHealSequence(path)
|
||||
if !exists {
|
||||
hsp = madmin.HealStopSuccess{
|
||||
ClientToken: "invalid",
|
||||
StartTime: UTCNow(),
|
||||
}
|
||||
} else {
|
||||
hsp = madmin.HealStopSuccess{
|
||||
ClientToken: he.clientToken,
|
||||
ClientAddress: he.clientAddress,
|
||||
StartTime: he.startTime,
|
||||
}
|
||||
|
||||
he.stop()
|
||||
for !he.hasEnded() {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
// Heal sequence explicitly stopped, remove it.
|
||||
delete(ahs.healSeqMap, path)
|
||||
}
|
||||
|
||||
b, err := json.Marshal(&hsp)
|
||||
return b, toAdminAPIErrCode(context.Background(), err)
|
||||
}
|
||||
|
||||
// LaunchNewHealSequence - launches a background routine that performs
|
||||
// healing according to the healSequence argument. For each heal
|
||||
// sequence, state is stored in the `globalAllHealState`, which is a
|
||||
@@ -143,20 +199,20 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
||||
existsAndLive = true
|
||||
}
|
||||
}
|
||||
|
||||
if existsAndLive {
|
||||
// A heal sequence exists on the given path.
|
||||
if h.forceStarted {
|
||||
// stop the running heal sequence - wait for
|
||||
// it to finish.
|
||||
// stop the running heal sequence - wait for it to finish.
|
||||
he.stop()
|
||||
for !he.hasEnded() {
|
||||
time.Sleep(10 * time.Second)
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
} else {
|
||||
errMsg = "Heal is already running on the given path " +
|
||||
"(use force-start option to stop and start afresh). " +
|
||||
fmt.Sprintf("The heal was started by IP %s at %s",
|
||||
h.clientAddress, h.startTime)
|
||||
fmt.Sprintf("The heal was started by IP %s at %s, token is %s",
|
||||
h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken)
|
||||
|
||||
return nil, ErrHealAlreadyRunning, errMsg
|
||||
}
|
||||
@@ -183,48 +239,13 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
||||
// Launch top-level background heal go-routine
|
||||
go h.healSequenceStart()
|
||||
|
||||
// Launch clean-up routine to remove this heal sequence (after
|
||||
// it ends) from the global state after timeout has elapsed.
|
||||
go func() {
|
||||
var keepStateTimeout <-chan time.Time
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
everyMinute := ticker.C
|
||||
for {
|
||||
select {
|
||||
// Check every minute if heal sequence has ended.
|
||||
case <-everyMinute:
|
||||
if h.hasEnded() {
|
||||
keepStateTimeout = time.After(keepHealSeqStateDuration)
|
||||
everyMinute = nil
|
||||
}
|
||||
|
||||
// This case does not fire until the heal
|
||||
// sequence completes.
|
||||
case <-keepStateTimeout:
|
||||
// Heal sequence has ended, keep
|
||||
// results state duration has elapsed,
|
||||
// so purge state.
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
delete(ahs.healSeqMap, h.path)
|
||||
return
|
||||
|
||||
case <-globalServiceDoneCh:
|
||||
// server could be restarting - need
|
||||
// to exit immediately
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
ClientToken: h.clientToken,
|
||||
ClientAddress: h.clientAddress,
|
||||
StartTime: h.startTime,
|
||||
})
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
logger.LogIf(h.ctx, err)
|
||||
return nil, ErrInternalError, ""
|
||||
}
|
||||
return b, ErrNone, ""
|
||||
@@ -272,7 +293,7 @@ func (ahs *allHealState) PopHealStatusJSON(path string,
|
||||
|
||||
jbytes, err := json.Marshal(h.currentStatus)
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
logger.LogIf(h.ctx, err)
|
||||
return nil, ErrInternalError
|
||||
}
|
||||
|
||||
@@ -285,12 +306,15 @@ type healSequence struct {
|
||||
// bucket, and prefix on which heal seq. was initiated
|
||||
bucket, objPrefix string
|
||||
|
||||
// path is just bucket + "/" + objPrefix
|
||||
// path is just pathJoin(bucket, objPrefix)
|
||||
path string
|
||||
|
||||
// time at which heal sequence was started
|
||||
startTime time.Time
|
||||
|
||||
// time at which heal sequence has ended
|
||||
endTime time.Time
|
||||
|
||||
// Heal client info
|
||||
clientToken, clientAddress string
|
||||
|
||||
@@ -330,7 +354,7 @@ func newHealSequence(bucket, objPrefix, clientAddr string,
|
||||
return &healSequence{
|
||||
bucket: bucket,
|
||||
objPrefix: objPrefix,
|
||||
path: bucket + "/" + objPrefix,
|
||||
path: pathJoin(bucket, objPrefix),
|
||||
startTime: UTCNow(),
|
||||
clientToken: mustGetUUID(),
|
||||
clientAddress: clientAddr,
|
||||
@@ -468,6 +492,7 @@ func (h *healSequence) healSequenceStart() {
|
||||
|
||||
select {
|
||||
case err, ok := <-h.traverseAndHealDoneCh:
|
||||
h.endTime = UTCNow()
|
||||
h.currentStatus.updateLock.Lock()
|
||||
defer h.currentStatus.updateLock.Unlock()
|
||||
// Heal traversal is complete.
|
||||
@@ -481,6 +506,7 @@ func (h *healSequence) healSequenceStart() {
|
||||
}
|
||||
|
||||
case <-h.stopSignalCh:
|
||||
h.endTime = UTCNow()
|
||||
h.currentStatus.updateLock.Lock()
|
||||
h.currentStatus.Summary = healStoppedStatus
|
||||
h.currentStatus.FailureDetail = errHealStopSignalled.Error()
|
||||
@@ -552,7 +578,7 @@ func (h *healSequence) healConfig() error {
|
||||
// before proceeding to heal
|
||||
waitCount := 60
|
||||
// Any requests in progress, delay the heal.
|
||||
for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
|
||||
for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
|
||||
waitCount--
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
@@ -562,7 +588,7 @@ func (h *healSequence) healConfig() error {
|
||||
objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, minioMetaBucket, minioConfigPrefix,
|
||||
marker, "", 1000)
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
|
||||
for index := range objectInfos.Objects {
|
||||
@@ -609,13 +635,18 @@ func (h *healSequence) healDiskFormat() error {
|
||||
// return any error, ignore error returned when disks have
|
||||
// already healed.
|
||||
if err != nil && err != errNoHealRequired {
|
||||
return errFnHealFromAPIErr(err)
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
|
||||
// Healing succeeded notify the peers to reload format and re-initialize disks.
|
||||
// We will not notify peers only if healing succeeded.
|
||||
if err == nil {
|
||||
peersReInitFormat(globalAdminPeers, h.settings.DryRun)
|
||||
for host, rerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
|
||||
if rerr != nil {
|
||||
logger.GetReqInfo(h.ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(h.ctx, rerr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Push format heal result
|
||||
@@ -641,7 +672,7 @@ func (h *healSequence) healBuckets() error {
|
||||
|
||||
buckets, err := objectAPI.ListBucketsHeal(h.ctx)
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
|
||||
for _, bucket := range buckets {
|
||||
@@ -698,7 +729,7 @@ func (h *healSequence) healBucket(bucket string) error {
|
||||
// before proceeding to heal
|
||||
waitCount := 60
|
||||
// Any requests in progress, delay the heal.
|
||||
for globalHTTPServer.GetRequestCount() > 0 && waitCount > 0 {
|
||||
for globalHTTPServer.GetRequestCount() > 2 && waitCount > 0 {
|
||||
waitCount--
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
@@ -708,7 +739,7 @@ func (h *healSequence) healBucket(bucket string) error {
|
||||
objectInfos, err := objectAPI.ListObjectsHeal(h.ctx, bucket,
|
||||
h.objPrefix, marker, "", entries)
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
return errFnHealFromAPIErr(h.ctx, err)
|
||||
}
|
||||
|
||||
g := errgroup.WithNErrs(len(objectInfos.Objects))
|
||||
|
||||
+52
-16
@@ -31,7 +31,7 @@ type adminAPIHandlers struct {
|
||||
}
|
||||
|
||||
// registerAdminRouter - Add handler functions for each service REST API routes.
|
||||
func registerAdminRouter(router *mux.Router) {
|
||||
func registerAdminRouter(router *mux.Router, enableIAM bool) {
|
||||
|
||||
adminAPI := adminAPIHandlers{}
|
||||
// Admin router
|
||||
@@ -53,12 +53,19 @@ func registerAdminRouter(router *mux.Router) {
|
||||
// Info operations
|
||||
adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
|
||||
|
||||
/// Heal operations
|
||||
if globalIsDistXL || globalIsXL {
|
||||
/// Heal operations
|
||||
|
||||
// Heal processing endpoint.
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
// Heal processing endpoint.
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
|
||||
|
||||
/// Health operations
|
||||
|
||||
// Performance command - return performance details based on input type
|
||||
adminV1Router.Methods(http.MethodGet).Path("/performance").HandlerFunc(httpTraceAll(adminAPI.PerfInfoHandler)).Queries("perfType", "{perfType:.*}")
|
||||
}
|
||||
|
||||
// Profiling operations
|
||||
adminV1Router.Methods(http.MethodPost).Path("/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
|
||||
@@ -67,15 +74,44 @@ func registerAdminRouter(router *mux.Router) {
|
||||
|
||||
/// Config operations
|
||||
|
||||
// Update credentials
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateCredentialsHandler))
|
||||
// Get config
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
|
||||
// Set config
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
|
||||
if enableIAM {
|
||||
// Update credentials
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
|
||||
// Get config
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
|
||||
// Set config
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
|
||||
|
||||
// Get config keys/values
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
|
||||
// Set config keys/values
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
|
||||
// Get config keys/values
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
|
||||
// Set config keys/values
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
|
||||
|
||||
// -- IAM APIs --
|
||||
|
||||
// Add policy IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
|
||||
|
||||
// Add user IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
|
||||
|
||||
// Remove policy IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
|
||||
|
||||
// Remove user IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
// List users
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
|
||||
|
||||
// List policies
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
|
||||
}
|
||||
|
||||
// If none of the routes match, return error.
|
||||
adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
|
||||
}
|
||||
|
||||
+5
-39
@@ -45,29 +45,12 @@ func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error)
|
||||
return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
|
||||
}
|
||||
|
||||
// ReInitFormat - re-initialize disk format, remotely.
|
||||
func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
|
||||
args := ReInitFormatArgs{DryRun: dryRun}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
|
||||
}
|
||||
|
||||
// ServerInfo - returns the server info of the server to which the RPC call is made.
|
||||
func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
|
||||
err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
|
||||
return sid, err
|
||||
}
|
||||
|
||||
// GetConfig - returns config.json of the remote server.
|
||||
func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
|
||||
args := AuthArgs{}
|
||||
var reply []byte
|
||||
|
||||
err := rpcClient.Call(adminServiceName+".GetConfig", &args, &reply)
|
||||
return reply, err
|
||||
}
|
||||
|
||||
// StartProfiling - starts profiling in the remote server.
|
||||
func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
|
||||
args := StartProfilingArgs{Profiler: profiler}
|
||||
@@ -125,9 +108,7 @@ func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
|
||||
// commands like service stop and service restart.
|
||||
type adminCmdRunner interface {
|
||||
SignalService(s serviceSignal) error
|
||||
ReInitFormat(dryRun bool) error
|
||||
ServerInfo() (ServerInfoData, error)
|
||||
GetConfig() ([]byte, error)
|
||||
StartProfiling(string) error
|
||||
DownloadProfilingData() ([]byte, error)
|
||||
}
|
||||
@@ -149,6 +130,11 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
// Use first IPv4 instead of loopback address.
|
||||
localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
|
||||
}
|
||||
if strings.HasPrefix(localAddr, "[::1]:") {
|
||||
// Use first IPv4 instead of loopback address.
|
||||
localAddr = net.JoinHostPort(localIP6.ToSlice()[0], globalMinioPort)
|
||||
}
|
||||
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
addr: localAddr,
|
||||
cmdRunner: localAdminClient{},
|
||||
@@ -169,26 +155,6 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
return adminPeerList
|
||||
}
|
||||
|
||||
// peersReInitFormat - reinitialize remote object layers to new format.
|
||||
func peersReInitFormat(peers adminPeers, dryRun bool) error {
|
||||
errs := make([]error, len(peers))
|
||||
|
||||
// Send ReInitFormat RPC call to all nodes.
|
||||
// for local adminPeer this is a no-op.
|
||||
wg := sync.WaitGroup{}
|
||||
for i, peer := range peers {
|
||||
wg.Add(1)
|
||||
go func(idx int, peer adminPeer) {
|
||||
defer wg.Done()
|
||||
if !peer.isLocal {
|
||||
errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
|
||||
}
|
||||
}(i, peer)
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Initialize global adminPeer collection.
|
||||
func initGlobalAdminPeers(endpoints EndpointList) {
|
||||
globalAdminPeers = makeAdminPeers(endpoints)
|
||||
|
||||
@@ -68,23 +68,6 @@ func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[
|
||||
return
|
||||
}
|
||||
|
||||
// GetConfig - returns the config.json of this server.
|
||||
func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
|
||||
*reply, err = receiver.local.GetConfig()
|
||||
return err
|
||||
}
|
||||
|
||||
// ReInitFormatArgs - provides dry-run information to re-initialize format.json
|
||||
type ReInitFormatArgs struct {
|
||||
AuthArgs
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// ReInitFormat - re-init 'format.json'
|
||||
func (receiver *adminRPCReceiver) ReInitFormat(args *ReInitFormatArgs, reply *VoidReply) error {
|
||||
return receiver.local.ReInitFormat(args.DryRun)
|
||||
}
|
||||
|
||||
// NewAdminRPCServer - returns new admin RPC server.
|
||||
func NewAdminRPCServer() (*xrpc.Server, error) {
|
||||
rpcServer := xrpc.NewServer()
|
||||
|
||||
+1
-79
@@ -40,11 +40,8 @@ import (
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalServiceSignalCh := globalServiceSignalCh
|
||||
defer func(sigChan chan serviceSignal) { globalServiceSignalCh = sigChan }(globalServiceSignalCh)
|
||||
globalServiceSignalCh = make(chan serviceSignal, 10)
|
||||
defer func() {
|
||||
globalServiceSignalCh = tmpGlobalServiceSignalCh
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
signal serviceSignal
|
||||
@@ -66,34 +63,6 @@ func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalObjectAPI := globalObjectAPI
|
||||
defer func() {
|
||||
globalObjectAPI = tmpGlobalObjectAPI
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
objectAPI ObjectLayer
|
||||
dryRun bool
|
||||
expectErr bool
|
||||
}{
|
||||
{&DummyObjectLayer{}, true, false},
|
||||
{&DummyObjectLayer{}, false, false},
|
||||
{nil, true, true},
|
||||
{nil, false, true},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
globalObjectAPI = testCase.objectAPI
|
||||
err := client.ReInitFormat(testCase.dryRun)
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalBootTime := globalBootTime
|
||||
tmpGlobalObjectAPI := globalObjectAPI
|
||||
@@ -140,33 +109,6 @@ func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerGetConfig(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalServerConfig := globalServerConfig
|
||||
defer func() {
|
||||
globalServerConfig = tmpGlobalServerConfig
|
||||
}()
|
||||
|
||||
config := newServerConfig()
|
||||
|
||||
testCases := []struct {
|
||||
config *serverConfig
|
||||
expectErr bool
|
||||
}{
|
||||
{globalServerConfig, false},
|
||||
{config, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
globalServerConfig = testCase.config
|
||||
_, err := client.GetConfig()
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
|
||||
rpcServer, err := NewAdminRPCServer()
|
||||
if err != nil {
|
||||
@@ -208,16 +150,6 @@ func TestAdminRPCClientSignalService(t *testing.T) {
|
||||
testAdminCmdRunnerSignalService(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientReInitFormat(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
defer func() {
|
||||
globalServerConfig = prevGlobalServerConfig
|
||||
}()
|
||||
|
||||
testAdminCmdRunnerReInitFormat(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientServerInfo(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
@@ -227,13 +159,3 @@ func TestAdminRPCClientServerInfo(t *testing.T) {
|
||||
|
||||
testAdminCmdRunnerServerInfo(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientGetConfig(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
defer func() {
|
||||
globalServerConfig = prevGlobalServerConfig
|
||||
}()
|
||||
|
||||
testAdminCmdRunnerGetConfig(t, rpcClient)
|
||||
}
|
||||
|
||||
+2
-84
@@ -23,92 +23,10 @@ import (
|
||||
const (
|
||||
// Response request id.
|
||||
responseRequestIDKey = "x-amz-request-id"
|
||||
// Deployment id.
|
||||
responseDeploymentIDKey = "x-minio-deployment-id"
|
||||
)
|
||||
|
||||
// CSVFileHeaderInfo -Can be either USE IGNORE OR NONE, defines what to do with
|
||||
// the first row
|
||||
type CSVFileHeaderInfo string
|
||||
|
||||
// Constants for file header info.
|
||||
const (
|
||||
CSVFileHeaderInfoNone CSVFileHeaderInfo = "NONE"
|
||||
CSVFileHeaderInfoIgnore = "IGNORE"
|
||||
CSVFileHeaderInfoUse = "USE"
|
||||
)
|
||||
|
||||
// SelectCompressionType - ONLY GZIP is supported
|
||||
type SelectCompressionType string
|
||||
|
||||
// Constants for compression types under select API.
|
||||
const (
|
||||
SelectCompressionNONE SelectCompressionType = "NONE"
|
||||
SelectCompressionGZIP = "GZIP"
|
||||
SelectCompressionBZIP = "BZIP2"
|
||||
)
|
||||
|
||||
// CSVQuoteFields - Can be either Always or AsNeeded
|
||||
type CSVQuoteFields string
|
||||
|
||||
// Constants for csv quote styles.
|
||||
const (
|
||||
CSVQuoteFieldsAlways CSVQuoteFields = "Always"
|
||||
CSVQuoteFieldsAsNeeded = "AsNeeded"
|
||||
)
|
||||
|
||||
// QueryExpressionType - Currently can only be SQL
|
||||
type QueryExpressionType string
|
||||
|
||||
// Constants for expression type.
|
||||
const (
|
||||
QueryExpressionTypeSQL QueryExpressionType = "SQL"
|
||||
)
|
||||
|
||||
// JSONType determines json input serialization type.
|
||||
type JSONType string
|
||||
|
||||
// Constants for JSONTypes.
|
||||
const (
|
||||
JSONDocumentType JSONType = "Document"
|
||||
JSONLinesType = "Lines"
|
||||
)
|
||||
|
||||
// ObjectSelectRequest - represents the input select body
|
||||
type ObjectSelectRequest struct {
|
||||
XMLName xml.Name `xml:"SelectObjectContentRequest" json:"-"`
|
||||
Expression string
|
||||
ExpressionType QueryExpressionType
|
||||
InputSerialization struct {
|
||||
CompressionType SelectCompressionType
|
||||
Parquet *struct{}
|
||||
CSV *struct {
|
||||
FileHeaderInfo CSVFileHeaderInfo
|
||||
RecordDelimiter string
|
||||
FieldDelimiter string
|
||||
QuoteCharacter string
|
||||
QuoteEscapeCharacter string
|
||||
Comments string
|
||||
}
|
||||
JSON *struct {
|
||||
Type JSONType
|
||||
}
|
||||
}
|
||||
OutputSerialization struct {
|
||||
CSV *struct {
|
||||
QuoteFields CSVQuoteFields
|
||||
RecordDelimiter string
|
||||
FieldDelimiter string
|
||||
QuoteCharacter string
|
||||
QuoteEscapeCharacter string
|
||||
}
|
||||
JSON *struct {
|
||||
RecordDelimiter string
|
||||
}
|
||||
}
|
||||
RequestProgress struct {
|
||||
Enabled bool
|
||||
}
|
||||
}
|
||||
|
||||
// ObjectIdentifier carries key name for the object to delete.
|
||||
type ObjectIdentifier struct {
|
||||
ObjectName string `xml:"Key"`
|
||||
|
||||
+65
-180
@@ -23,11 +23,11 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/s3select"
|
||||
)
|
||||
|
||||
// APIError structure
|
||||
@@ -85,6 +85,7 @@ const (
|
||||
ErrNoSuchBucketPolicy
|
||||
ErrNoSuchKey
|
||||
ErrNoSuchUpload
|
||||
ErrNoSuchVersion
|
||||
ErrNotImplemented
|
||||
ErrPreconditionFailed
|
||||
ErrRequestTimeTooSkewed
|
||||
@@ -150,6 +151,9 @@ const (
|
||||
ErrKMSNotConfigured
|
||||
ErrKMSAuthFailure
|
||||
|
||||
ErrNoAccessKey
|
||||
ErrInvalidToken
|
||||
|
||||
// Bucket notification related errors.
|
||||
ErrEventNotification
|
||||
ErrARNNotification
|
||||
@@ -187,11 +191,15 @@ const (
|
||||
// new error codes here.
|
||||
|
||||
ErrMalformedJSON
|
||||
ErrAdminNoSuchUser
|
||||
ErrAdminNoSuchPolicy
|
||||
ErrAdminInvalidArgument
|
||||
ErrAdminInvalidAccessKey
|
||||
ErrAdminInvalidSecretKey
|
||||
ErrAdminConfigNoQuorum
|
||||
ErrAdminConfigTooLarge
|
||||
ErrAdminConfigBadJSON
|
||||
ErrAdminConfigDuplicateKeys
|
||||
ErrAdminCredentialsMismatch
|
||||
ErrInsecureClientRequest
|
||||
ErrObjectTampered
|
||||
@@ -249,7 +257,6 @@ const (
|
||||
ErrParseUnsupportedAlias
|
||||
ErrParseUnsupportedSyntax
|
||||
ErrParseUnknownOperator
|
||||
ErrParseInvalidPathComponent
|
||||
ErrParseMissingIdentAfterAt
|
||||
ErrParseUnexpectedOperator
|
||||
ErrParseUnexpectedTerm
|
||||
@@ -288,8 +295,9 @@ const (
|
||||
ErrEvaluatorInvalidTimestampFormatPatternToken
|
||||
ErrEvaluatorInvalidTimestampFormatPatternSymbol
|
||||
ErrEvaluatorBindingDoesNotExist
|
||||
ErrInvalidColumnIndex
|
||||
ErrMissingHeaders
|
||||
ErrInvalidColumnIndex
|
||||
|
||||
ErrAdminConfigNotificationTargetsFailed
|
||||
ErrAdminProfilerNotEnabled
|
||||
ErrInvalidDecompressedSize
|
||||
@@ -438,6 +446,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNoSuchVersion: {
|
||||
Code: "NoSuchVersion",
|
||||
Description: "Indicates that the version ID specified in the request does not match an existing version.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNotImplemented: {
|
||||
Code: "NotImplemented",
|
||||
Description: "A header you provided implies functionality that is not implemented",
|
||||
@@ -804,6 +817,16 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Server side encryption specified but KMS authorization failed",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrNoAccessKey: {
|
||||
Code: "AccessDenied",
|
||||
Description: "No AWSAccessKey was presented",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
ErrInvalidToken: {
|
||||
Code: "InvalidTokenId",
|
||||
Description: "The security token included in the request is invalid",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
|
||||
/// S3 extensions.
|
||||
ErrContentSHA256Mismatch: {
|
||||
@@ -863,6 +886,21 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "The JSON you provided was not well-formed or did not validate against our published format.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminNoSuchUser: {
|
||||
Code: "XMinioAdminNoSuchUser",
|
||||
Description: "The specified user does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminNoSuchPolicy: {
|
||||
Code: "XMinioAdminNoSuchPolicy",
|
||||
Description: "The canned policy does not exist.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminInvalidArgument: {
|
||||
Code: "XMinioAdminInvalidArgument",
|
||||
Description: "Invalid arguments specified.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminInvalidAccessKey: {
|
||||
Code: "XMinioAdminInvalidAccessKey",
|
||||
Description: "The access key is invalid.",
|
||||
@@ -881,11 +919,16 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
ErrAdminConfigTooLarge: {
|
||||
Code: "XMinioAdminConfigTooLarge",
|
||||
Description: fmt.Sprintf("Configuration data provided exceeds the allowed maximum of %d bytes",
|
||||
maxConfigJSONSize),
|
||||
maxEConfigJSONSize),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigBadJSON: {
|
||||
Code: "XMinioAdminConfigBadJSON",
|
||||
Description: "JSON configuration provided is of incorrect format",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigDuplicateKeys: {
|
||||
Code: "XMinioAdminConfigDuplicateKeys",
|
||||
Description: "JSON configuration provided has objects with duplicate keys",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
@@ -1199,11 +1242,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "The SQL expression contains an invalid operator.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrParseInvalidPathComponent: {
|
||||
Code: "ParseInvalidPathComponent",
|
||||
Description: "The SQL expression contains an invalid path component.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrParseMissingIdentAfterAt: {
|
||||
Code: "ParseMissingIdentAfterAt",
|
||||
Description: "Did not find the expected identifier after the @ symbol in the SQL expression.",
|
||||
@@ -1389,11 +1427,6 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Time stamp format pattern contains an invalid symbol in the SQL expression.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidColumnIndex: {
|
||||
Code: "InvalidColumnIndex",
|
||||
Description: "Column index in the SQL expression is invalid.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrEvaluatorBindingDoesNotExist: {
|
||||
Code: "ErrEvaluatorBindingDoesNotExist",
|
||||
Description: "A column name or a path provided does not exist in the SQL expression",
|
||||
@@ -1404,6 +1437,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Some headers in the query are missing from the file. Check the file and try again.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidColumnIndex: {
|
||||
Code: "InvalidColumnIndex",
|
||||
Description: "The column index is invalid. Please check the service documentation and try again.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidDecompressedSize: {
|
||||
Code: "XMinioInvalidDecompressedSize",
|
||||
Description: "The data provided is unfit for decompression",
|
||||
@@ -1415,13 +1453,19 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
// toAPIErrorCode - Converts embedded errors. Convenience
|
||||
// function written to handle all cases where we have known types of
|
||||
// errors returned by underlying layers.
|
||||
func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
if err == nil {
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// Verify if the underlying error is signature mismatch.
|
||||
switch err {
|
||||
case errInvalidArgument:
|
||||
apiErr = ErrAdminInvalidArgument
|
||||
case errNoSuchUser:
|
||||
apiErr = ErrAdminNoSuchUser
|
||||
case errNoSuchPolicy:
|
||||
apiErr = ErrAdminNoSuchPolicy
|
||||
case errSignatureMismatch:
|
||||
apiErr = ErrSignatureDoesNotMatch
|
||||
case errInvalidRange:
|
||||
@@ -1435,10 +1479,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
case auth.ErrInvalidSecretKeyLength:
|
||||
apiErr = ErrAdminInvalidSecretKey
|
||||
// SSE errors
|
||||
case errInvalidEncryptionParameters:
|
||||
apiErr = ErrInvalidEncryptionParameters
|
||||
case crypto.ErrInvalidEncryptionMethod:
|
||||
apiErr = ErrInvalidEncryptionMethod
|
||||
case errInsecureSSERequest:
|
||||
apiErr = ErrInsecureSSECustomerRequest
|
||||
case crypto.ErrInvalidCustomerAlgorithm:
|
||||
apiErr = ErrInvalidSSECustomerAlgorithm
|
||||
case crypto.ErrInvalidCustomerKey:
|
||||
@@ -1466,169 +1510,6 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
case errOperationTimedOut, context.Canceled, context.DeadlineExceeded:
|
||||
apiErr = ErrOperationTimedOut
|
||||
}
|
||||
switch err {
|
||||
case s3select.ErrBusy:
|
||||
apiErr = ErrBusy
|
||||
case s3select.ErrUnauthorizedAccess:
|
||||
apiErr = ErrUnauthorizedAccess
|
||||
case s3select.ErrExpressionTooLong:
|
||||
apiErr = ErrExpressionTooLong
|
||||
case s3select.ErrIllegalSQLFunctionArgument:
|
||||
apiErr = ErrIllegalSQLFunctionArgument
|
||||
case s3select.ErrInvalidKeyPath:
|
||||
apiErr = ErrInvalidKeyPath
|
||||
case s3select.ErrInvalidCompressionFormat:
|
||||
apiErr = ErrInvalidCompressionFormat
|
||||
case s3select.ErrInvalidFileHeaderInfo:
|
||||
apiErr = ErrInvalidFileHeaderInfo
|
||||
case s3select.ErrInvalidJSONType:
|
||||
apiErr = ErrInvalidJSONType
|
||||
case s3select.ErrInvalidQuoteFields:
|
||||
apiErr = ErrInvalidQuoteFields
|
||||
case s3select.ErrInvalidRequestParameter:
|
||||
apiErr = ErrInvalidRequestParameter
|
||||
case s3select.ErrInvalidDataType:
|
||||
apiErr = ErrInvalidDataType
|
||||
case s3select.ErrInvalidTextEncoding:
|
||||
apiErr = ErrInvalidTextEncoding
|
||||
case s3select.ErrInvalidTableAlias:
|
||||
apiErr = ErrInvalidTableAlias
|
||||
case s3select.ErrMissingRequiredParameter:
|
||||
apiErr = ErrMissingRequiredParameter
|
||||
case s3select.ErrObjectSerializationConflict:
|
||||
apiErr = ErrObjectSerializationConflict
|
||||
case s3select.ErrUnsupportedSQLOperation:
|
||||
apiErr = ErrUnsupportedSQLOperation
|
||||
case s3select.ErrUnsupportedSQLStructure:
|
||||
apiErr = ErrUnsupportedSQLStructure
|
||||
case s3select.ErrUnsupportedSyntax:
|
||||
apiErr = ErrUnsupportedSyntax
|
||||
case s3select.ErrUnsupportedRangeHeader:
|
||||
apiErr = ErrUnsupportedRangeHeader
|
||||
case s3select.ErrLexerInvalidChar:
|
||||
apiErr = ErrLexerInvalidChar
|
||||
case s3select.ErrLexerInvalidOperator:
|
||||
apiErr = ErrLexerInvalidOperator
|
||||
case s3select.ErrLexerInvalidLiteral:
|
||||
apiErr = ErrLexerInvalidLiteral
|
||||
case s3select.ErrLexerInvalidIONLiteral:
|
||||
apiErr = ErrLexerInvalidIONLiteral
|
||||
case s3select.ErrParseExpectedDatePart:
|
||||
apiErr = ErrParseExpectedDatePart
|
||||
case s3select.ErrParseExpectedKeyword:
|
||||
apiErr = ErrParseExpectedKeyword
|
||||
case s3select.ErrParseExpectedTokenType:
|
||||
apiErr = ErrParseExpectedTokenType
|
||||
case s3select.ErrParseExpected2TokenTypes:
|
||||
apiErr = ErrParseExpected2TokenTypes
|
||||
case s3select.ErrParseExpectedNumber:
|
||||
apiErr = ErrParseExpectedNumber
|
||||
case s3select.ErrParseExpectedRightParenBuiltinFunctionCall:
|
||||
apiErr = ErrParseExpectedRightParenBuiltinFunctionCall
|
||||
case s3select.ErrParseExpectedTypeName:
|
||||
apiErr = ErrParseExpectedTypeName
|
||||
case s3select.ErrParseExpectedWhenClause:
|
||||
apiErr = ErrParseExpectedWhenClause
|
||||
case s3select.ErrParseUnsupportedToken:
|
||||
apiErr = ErrParseUnsupportedToken
|
||||
case s3select.ErrParseUnsupportedLiteralsGroupBy:
|
||||
apiErr = ErrParseUnsupportedLiteralsGroupBy
|
||||
case s3select.ErrParseExpectedMember:
|
||||
apiErr = ErrParseExpectedMember
|
||||
case s3select.ErrParseUnsupportedSelect:
|
||||
apiErr = ErrParseUnsupportedSelect
|
||||
case s3select.ErrParseUnsupportedCase:
|
||||
apiErr = ErrParseUnsupportedCase
|
||||
case s3select.ErrParseUnsupportedCaseClause:
|
||||
apiErr = ErrParseUnsupportedCaseClause
|
||||
case s3select.ErrParseUnsupportedAlias:
|
||||
apiErr = ErrParseUnsupportedAlias
|
||||
case s3select.ErrParseUnsupportedSyntax:
|
||||
apiErr = ErrParseUnsupportedSyntax
|
||||
case s3select.ErrParseUnknownOperator:
|
||||
apiErr = ErrParseUnknownOperator
|
||||
case s3select.ErrParseInvalidPathComponent:
|
||||
apiErr = ErrParseInvalidPathComponent
|
||||
case s3select.ErrParseMissingIdentAfterAt:
|
||||
apiErr = ErrParseMissingIdentAfterAt
|
||||
case s3select.ErrParseUnexpectedOperator:
|
||||
apiErr = ErrParseUnexpectedOperator
|
||||
case s3select.ErrParseUnexpectedTerm:
|
||||
apiErr = ErrParseUnexpectedTerm
|
||||
case s3select.ErrParseUnexpectedToken:
|
||||
apiErr = ErrParseUnexpectedToken
|
||||
case s3select.ErrParseUnexpectedKeyword:
|
||||
apiErr = ErrParseUnexpectedKeyword
|
||||
case s3select.ErrParseExpectedExpression:
|
||||
apiErr = ErrParseExpectedExpression
|
||||
case s3select.ErrParseExpectedLeftParenAfterCast:
|
||||
apiErr = ErrParseExpectedLeftParenAfterCast
|
||||
case s3select.ErrParseExpectedLeftParenValueConstructor:
|
||||
apiErr = ErrParseExpectedLeftParenValueConstructor
|
||||
case s3select.ErrParseExpectedLeftParenBuiltinFunctionCall:
|
||||
apiErr = ErrParseExpectedLeftParenBuiltinFunctionCall
|
||||
case s3select.ErrParseExpectedArgumentDelimiter:
|
||||
apiErr = ErrParseExpectedArgumentDelimiter
|
||||
case s3select.ErrParseCastArity:
|
||||
apiErr = ErrParseCastArity
|
||||
case s3select.ErrParseInvalidTypeParam:
|
||||
apiErr = ErrParseInvalidTypeParam
|
||||
case s3select.ErrParseEmptySelect:
|
||||
apiErr = ErrParseEmptySelect
|
||||
case s3select.ErrParseSelectMissingFrom:
|
||||
apiErr = ErrParseSelectMissingFrom
|
||||
case s3select.ErrParseExpectedIdentForGroupName:
|
||||
apiErr = ErrParseExpectedIdentForGroupName
|
||||
case s3select.ErrParseExpectedIdentForAlias:
|
||||
apiErr = ErrParseExpectedIdentForAlias
|
||||
case s3select.ErrParseUnsupportedCallWithStar:
|
||||
apiErr = ErrParseUnsupportedCallWithStar
|
||||
case s3select.ErrParseNonUnaryAgregateFunctionCall:
|
||||
apiErr = ErrParseNonUnaryAgregateFunctionCall
|
||||
case s3select.ErrParseMalformedJoin:
|
||||
apiErr = ErrParseMalformedJoin
|
||||
case s3select.ErrParseExpectedIdentForAt:
|
||||
apiErr = ErrParseExpectedIdentForAt
|
||||
case s3select.ErrParseAsteriskIsNotAloneInSelectList:
|
||||
apiErr = ErrParseAsteriskIsNotAloneInSelectList
|
||||
case s3select.ErrParseCannotMixSqbAndWildcardInSelectList:
|
||||
apiErr = ErrParseCannotMixSqbAndWildcardInSelectList
|
||||
case s3select.ErrParseInvalidContextForWildcardInSelectList:
|
||||
apiErr = ErrParseInvalidContextForWildcardInSelectList
|
||||
case s3select.ErrIncorrectSQLFunctionArgumentType:
|
||||
apiErr = ErrIncorrectSQLFunctionArgumentType
|
||||
case s3select.ErrValueParseFailure:
|
||||
apiErr = ErrValueParseFailure
|
||||
case s3select.ErrIntegerOverflow:
|
||||
apiErr = ErrIntegerOverflow
|
||||
case s3select.ErrLikeInvalidInputs:
|
||||
apiErr = ErrLikeInvalidInputs
|
||||
case s3select.ErrCastFailed:
|
||||
apiErr = ErrCastFailed
|
||||
case s3select.ErrInvalidCast:
|
||||
apiErr = ErrInvalidCast
|
||||
case s3select.ErrEvaluatorInvalidTimestampFormatPattern:
|
||||
apiErr = ErrEvaluatorInvalidTimestampFormatPattern
|
||||
case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing:
|
||||
apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing
|
||||
case s3select.ErrEvaluatorTimestampFormatPatternDuplicateFields:
|
||||
apiErr = ErrEvaluatorTimestampFormatPatternDuplicateFields
|
||||
case s3select.ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch:
|
||||
apiErr = ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch
|
||||
case s3select.ErrEvaluatorUnterminatedTimestampFormatPatternToken:
|
||||
apiErr = ErrEvaluatorUnterminatedTimestampFormatPatternToken
|
||||
case s3select.ErrEvaluatorInvalidTimestampFormatPatternToken:
|
||||
apiErr = ErrEvaluatorInvalidTimestampFormatPatternToken
|
||||
case s3select.ErrEvaluatorInvalidTimestampFormatPatternSymbol:
|
||||
apiErr = ErrEvaluatorInvalidTimestampFormatPatternSymbol
|
||||
case s3select.ErrInvalidColumnIndex:
|
||||
apiErr = ErrInvalidColumnIndex
|
||||
case s3select.ErrEvaluatorBindingDoesNotExist:
|
||||
apiErr = ErrEvaluatorBindingDoesNotExist
|
||||
case s3select.ErrMissingHeaders:
|
||||
apiErr = ErrMissingHeaders
|
||||
|
||||
}
|
||||
|
||||
// Compression errors
|
||||
switch err {
|
||||
@@ -1740,6 +1621,10 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrObjectTampered
|
||||
default:
|
||||
apiErr = ErrInternalError
|
||||
// Make sure to log the errors which we cannot translate
|
||||
// to a meaningful S3 API errors. This is added to aid in
|
||||
// debugging unexpected/unhandled errors.
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
|
||||
return apiErr
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
|
||||
@@ -52,7 +53,6 @@ var toAPIErrorCodeTests = []struct {
|
||||
{err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch},
|
||||
|
||||
// SSE-C errors
|
||||
{err: errInsecureSSERequest, errCode: ErrInsecureSSECustomerRequest},
|
||||
{err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm},
|
||||
{err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey},
|
||||
{err: crypto.ErrInvalidCustomerKey, errCode: ErrInvalidSSECustomerKey},
|
||||
@@ -65,8 +65,9 @@ var toAPIErrorCodeTests = []struct {
|
||||
}
|
||||
|
||||
func TestAPIErrCode(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
for i, testCase := range toAPIErrorCodeTests {
|
||||
errCode := toAPIErrorCode(testCase.err)
|
||||
errCode := toAPIErrorCode(ctx, testCase.err)
|
||||
if errCode != testCase.errCode {
|
||||
t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode)
|
||||
}
|
||||
|
||||
+52
-20
@@ -22,15 +22,22 @@ import (
|
||||
)
|
||||
|
||||
// Parse bucket url queries
|
||||
func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string) {
|
||||
prefix = values.Get("prefix")
|
||||
marker = values.Get("marker")
|
||||
delimiter = values.Get("delimiter")
|
||||
func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string, errCode APIErrorCode) {
|
||||
errCode = ErrNone
|
||||
|
||||
if values.Get("max-keys") != "" {
|
||||
maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
|
||||
var err error
|
||||
if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
|
||||
errCode = ErrInvalidMaxKeys
|
||||
return
|
||||
}
|
||||
} else {
|
||||
maxkeys = maxObjectList
|
||||
}
|
||||
|
||||
prefix = values.Get("prefix")
|
||||
marker = values.Get("marker")
|
||||
delimiter = values.Get("delimiter")
|
||||
encodingType = values.Get("encoding-type")
|
||||
return
|
||||
}
|
||||
@@ -47,44 +54,69 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
|
||||
}
|
||||
}
|
||||
|
||||
if values.Get("max-keys") != "" {
|
||||
var err error
|
||||
if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil {
|
||||
errCode = ErrInvalidMaxKeys
|
||||
return
|
||||
}
|
||||
} else {
|
||||
maxkeys = maxObjectList
|
||||
}
|
||||
|
||||
prefix = values.Get("prefix")
|
||||
token = values.Get("continuation-token")
|
||||
startAfter = values.Get("start-after")
|
||||
delimiter = values.Get("delimiter")
|
||||
if values.Get("max-keys") != "" {
|
||||
maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
|
||||
} else {
|
||||
maxkeys = maxObjectList
|
||||
}
|
||||
fetchOwner = values.Get("fetch-owner") == "true"
|
||||
encodingType = values.Get("encoding-type")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse bucket url queries for ?uploads
|
||||
func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string) {
|
||||
func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string, errCode APIErrorCode) {
|
||||
errCode = ErrNone
|
||||
|
||||
if values.Get("max-uploads") != "" {
|
||||
var err error
|
||||
if maxUploads, err = strconv.Atoi(values.Get("max-uploads")); err != nil {
|
||||
errCode = ErrInvalidMaxUploads
|
||||
return
|
||||
}
|
||||
} else {
|
||||
maxUploads = maxUploadsList
|
||||
}
|
||||
|
||||
prefix = values.Get("prefix")
|
||||
keyMarker = values.Get("key-marker")
|
||||
uploadIDMarker = values.Get("upload-id-marker")
|
||||
delimiter = values.Get("delimiter")
|
||||
if values.Get("max-uploads") != "" {
|
||||
maxUploads, _ = strconv.Atoi(values.Get("max-uploads"))
|
||||
} else {
|
||||
maxUploads = maxUploadsList
|
||||
}
|
||||
encodingType = values.Get("encoding-type")
|
||||
return
|
||||
}
|
||||
|
||||
// Parse object url queries
|
||||
func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string) {
|
||||
uploadID = values.Get("uploadId")
|
||||
partNumberMarker, _ = strconv.Atoi(values.Get("part-number-marker"))
|
||||
func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string, errCode APIErrorCode) {
|
||||
var err error
|
||||
errCode = ErrNone
|
||||
|
||||
if values.Get("max-parts") != "" {
|
||||
maxParts, _ = strconv.Atoi(values.Get("max-parts"))
|
||||
if maxParts, err = strconv.Atoi(values.Get("max-parts")); err != nil {
|
||||
errCode = ErrInvalidMaxParts
|
||||
return
|
||||
}
|
||||
} else {
|
||||
maxParts = maxPartsList
|
||||
}
|
||||
|
||||
if values.Get("part-number-marker") != "" {
|
||||
if partNumberMarker, err = strconv.Atoi(values.Get("part-number-marker")); err != nil {
|
||||
errCode = ErrInvalidPartNumberMarker
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
uploadID = values.Get("uploadId")
|
||||
encodingType = values.Get("encoding-type")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -156,7 +156,10 @@ func TestListObjectsV1Resources(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
prefix, marker, delimiter, maxKeys, encodingType := getListObjectsV1Args(testCase.values)
|
||||
prefix, marker, delimiter, maxKeys, encodingType, argsErr := getListObjectsV1Args(testCase.values)
|
||||
if argsErr != ErrNone {
|
||||
t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
|
||||
}
|
||||
if prefix != testCase.prefix {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix)
|
||||
}
|
||||
@@ -198,7 +201,10 @@ func TestGetObjectsResources(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
uploadID, partNumberMarker, maxParts, encodingType := getObjectResources(testCase.values)
|
||||
uploadID, partNumberMarker, maxParts, encodingType, argsErr := getObjectResources(testCase.values)
|
||||
if argsErr != ErrNone {
|
||||
t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr)
|
||||
}
|
||||
if uploadID != testCase.uploadID {
|
||||
t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID)
|
||||
}
|
||||
|
||||
@@ -16,7 +16,10 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "net/http"
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// Represents additional fields necessary for ErrPartTooSmall S3 error.
|
||||
type completeMultipartAPIError struct {
|
||||
@@ -42,7 +45,7 @@ type completeMultipartAPIError struct {
|
||||
// of this function.
|
||||
func writePartSmallErrorResponse(w http.ResponseWriter, r *http.Request, err PartTooSmall) {
|
||||
|
||||
apiError := getAPIError(toAPIErrorCode(err))
|
||||
apiError := getAPIError(toAPIErrorCode(context.Background(), err))
|
||||
// Generate complete multipart error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, r.URL.Path, w.Header().Get(responseRequestIDKey))
|
||||
cmpErrResp := completeMultipartAPIError{err.PartSize, int64(5242880), err.PartNumber, err.PartETag, errorResponse}
|
||||
|
||||
+11
-2
@@ -568,13 +568,22 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
|
||||
}
|
||||
|
||||
// writeErrorRespone writes error headers
|
||||
func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
|
||||
func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL, browser bool) {
|
||||
switch errorCode {
|
||||
case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set("Retry-After", "120")
|
||||
case ErrAccessDenied:
|
||||
// The request is from browser and also if browser
|
||||
// is enabled we need to redirect.
|
||||
if browser && globalIsBrowserEnabled {
|
||||
w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
|
||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
apiError := getAPIError(errorCode)
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
|
||||
@@ -608,7 +617,7 @@ func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
|
||||
Code: apiError.Code,
|
||||
Message: errBody,
|
||||
Resource: reqURL.Path,
|
||||
RequestID: "3L137",
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
HostID: "3L137",
|
||||
}
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
|
||||
+6
-1
@@ -26,14 +26,19 @@ import (
|
||||
type objectAPIHandlers struct {
|
||||
ObjectAPI func() ObjectLayer
|
||||
CacheAPI func() CacheObjectLayer
|
||||
// Returns true of handlers should interpret encryption.
|
||||
EncryptionEnabled func() bool
|
||||
}
|
||||
|
||||
// registerAPIRouter - registers S3 compatible APIs.
|
||||
func registerAPIRouter(router *mux.Router) {
|
||||
func registerAPIRouter(router *mux.Router, encryptionEnabled bool) {
|
||||
// Initialize API.
|
||||
api := objectAPIHandlers{
|
||||
ObjectAPI: newObjectLayerFn,
|
||||
CacheAPI: newCacheObjectsFn,
|
||||
EncryptionEnabled: func() bool {
|
||||
return encryptionEnabled
|
||||
},
|
||||
}
|
||||
|
||||
// API Router
|
||||
|
||||
+199
-26
@@ -19,6 +19,7 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
@@ -27,8 +28,11 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
jwtgo "github.com/dgrijalva/jwt-go"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
|
||||
@@ -86,6 +90,7 @@ const (
|
||||
authTypeSigned
|
||||
authTypeSignedV2
|
||||
authTypeJWT
|
||||
authTypeSTS
|
||||
)
|
||||
|
||||
// Get request authentication type.
|
||||
@@ -104,6 +109,8 @@ func getRequestAuthType(r *http.Request) authType {
|
||||
return authTypeJWT
|
||||
} else if isRequestPostPolicySignatureV4(r) {
|
||||
return authTypePostPolicy
|
||||
} else if _, ok := r.URL.Query()["Action"]; ok {
|
||||
return authTypeSTS
|
||||
} else if _, ok := r.Header["Authorization"]; !ok {
|
||||
return authTypeAnonymous
|
||||
}
|
||||
@@ -112,43 +119,140 @@ func getRequestAuthType(r *http.Request) authType {
|
||||
|
||||
// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
|
||||
// It does not accept presigned or JWT or anonymous requests.
|
||||
func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
|
||||
func checkAdminRequestAuthType(ctx context.Context, r *http.Request, region string) APIErrorCode {
|
||||
s3Err := ErrAccessDenied
|
||||
if _, ok := r.Header["X-Amz-Content-Sha256"]; ok && getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) { // we only support V4 (no presign) with auth. body
|
||||
s3Err = isReqAuthenticated(r, region)
|
||||
if _, ok := r.Header["X-Amz-Content-Sha256"]; ok &&
|
||||
getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
|
||||
// We only support admin credentials to access admin APIs.
|
||||
|
||||
var owner bool
|
||||
_, owner, s3Err = getReqAccessKeyV4(r, region)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
if !owner {
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// we only support V4 (no presign) with auth body
|
||||
s3Err = isReqAuthenticated(ctx, r, region)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
ctx := logger.SetReqInfo(ctx, reqInfo)
|
||||
logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description))
|
||||
}
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) APIErrorCode {
|
||||
isOwner := true
|
||||
accountName := globalServerConfig.GetCredential().AccessKey
|
||||
// Fetch the security token set by the client.
|
||||
func getSessionToken(r *http.Request) (token string) {
|
||||
token = r.Header.Get("X-Amz-Security-Token")
|
||||
if token != "" {
|
||||
return token
|
||||
}
|
||||
return r.URL.Query().Get("X-Amz-Security-Token")
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client, doesn't return
|
||||
// errors - upon errors the returned claims map will be empty.
|
||||
func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
|
||||
claims, _ := getClaimsFromToken(r)
|
||||
return claims
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client.
|
||||
func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
|
||||
claims := make(map[string]interface{})
|
||||
token := getSessionToken(r)
|
||||
if token == "" {
|
||||
return claims, nil
|
||||
}
|
||||
stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
|
||||
// JWT token for x-amz-security-token is signed with admin
|
||||
// secret key, temporary credentials become invalid if
|
||||
// server admin credentials change. This is done to ensure
|
||||
// that clients cannot decode the token using the temp
|
||||
// secret keys and generate an entirely new claim by essentially
|
||||
// hijacking the policies. We need to make sure that this is
|
||||
// based an admin credential such that token cannot be decoded
|
||||
// on the client side and is treated like an opaque value.
|
||||
return []byte(globalServerConfig.GetCredential().SecretKey), nil
|
||||
}
|
||||
p := &jwtgo.Parser{
|
||||
ValidMethods: []string{
|
||||
jwtgo.SigningMethodHS256.Alg(),
|
||||
jwtgo.SigningMethodHS512.Alg(),
|
||||
},
|
||||
}
|
||||
jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !jtoken.Valid {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
v, ok := claims["accessKey"]
|
||||
if !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
if _, ok = v.(string); !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client and validate the token.
|
||||
func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
|
||||
token := getSessionToken(r)
|
||||
if token != "" && cred.AccessKey == "" {
|
||||
return nil, ErrNoAccessKey
|
||||
}
|
||||
if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
|
||||
return nil, ErrInvalidToken
|
||||
}
|
||||
claims, err := getClaimsFromToken(r)
|
||||
if err != nil {
|
||||
return nil, toAPIErrorCode(context.Background(), err)
|
||||
}
|
||||
return claims, ErrNone
|
||||
}
|
||||
|
||||
// Check request auth type verifies the incoming http request
|
||||
// - validates the request signature
|
||||
// - validates the policy action if anonymous tests bucket policies if any,
|
||||
// for authenticated requests validates IAM policies.
|
||||
// returns APIErrorCode if any to be replied to the client.
|
||||
func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
switch getRequestAuthType(r) {
|
||||
case authTypeUnknown:
|
||||
case authTypeUnknown, authTypeStreamingSigned:
|
||||
return ErrAccessDenied
|
||||
case authTypePresignedV2, authTypeSignedV2:
|
||||
if errorCode := isReqAuthenticatedV2(r); errorCode != ErrNone {
|
||||
return errorCode
|
||||
if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypeSigned, authTypePresigned:
|
||||
region := globalServerConfig.GetRegion()
|
||||
switch action {
|
||||
case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
|
||||
region = ""
|
||||
}
|
||||
|
||||
if errorCode := isReqAuthenticated(r, region); errorCode != ErrNone {
|
||||
return errorCode
|
||||
if s3Err = isReqAuthenticated(ctx, r, region); s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
default:
|
||||
isOwner = false
|
||||
accountName = ""
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
claims, s3Err := checkClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// LocationConstraint is valid only for CreateBucketAction.
|
||||
@@ -174,17 +278,31 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
|
||||
}
|
||||
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: accountName,
|
||||
Action: action,
|
||||
if cred.AccessKey == "" {
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: action,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, locationConstraint),
|
||||
IsOwner: false,
|
||||
ObjectName: objectName,
|
||||
}) {
|
||||
return ErrNone
|
||||
}
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: iampolicy.Action(action),
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, locationConstraint),
|
||||
IsOwner: isOwner,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
ObjectName: objectName,
|
||||
IsOwner: owner,
|
||||
Claims: claims,
|
||||
}) {
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
@@ -209,7 +327,7 @@ func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode)
|
||||
}
|
||||
|
||||
// Verify if request has valid AWS Signature Version '4'.
|
||||
func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
|
||||
func isReqAuthenticated(ctx context.Context, r *http.Request, region string) (s3Error APIErrorCode) {
|
||||
if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
|
||||
return errCode
|
||||
}
|
||||
@@ -246,7 +364,7 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
|
||||
// The verification happens implicit during reading.
|
||||
reader, err := hash.NewReader(r.Body, -1, hex.EncodeToString(contentMD5), hex.EncodeToString(contentSHA256), -1)
|
||||
if err != nil {
|
||||
return toAPIErrorCode(err)
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
r.Body = ioutil.NopCloser(reader)
|
||||
return ErrNone
|
||||
@@ -288,12 +406,67 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
} else if aType == authTypeJWT {
|
||||
// Validate Authorization header if its valid for JWT request.
|
||||
if !isHTTPRequestValid(r) {
|
||||
if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
a.handler.ServeHTTP(w, r)
|
||||
return
|
||||
} else if aType == authTypeSTS {
|
||||
a.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
|
||||
writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
|
||||
// isPutAllowed - check if PUT operation is allowed on the resource, this
|
||||
// call verifies bucket policies and IAM policies, supports multi user
|
||||
// checks etc.
|
||||
func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request) (s3Err APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
switch atype {
|
||||
case authTypeUnknown:
|
||||
return ErrAccessDenied
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
cred, owner, s3Err = getReqAccessKeyV2(r)
|
||||
case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
|
||||
region := globalServerConfig.GetRegion()
|
||||
cred, owner, s3Err = getReqAccessKeyV4(r, region)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
claims, s3Err := checkClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
if cred.AccessKey == "" {
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: policy.PutObjectAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: false,
|
||||
ObjectName: objectName,
|
||||
}) {
|
||||
return ErrNone
|
||||
}
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Action: policy.PutObjectAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
ObjectName: objectName,
|
||||
IsOwner: owner,
|
||||
Claims: claims,
|
||||
}) {
|
||||
return ErrNone
|
||||
}
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@@ -377,11 +378,12 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
// Validates all testcases.
|
||||
for i, testCase := range testCases {
|
||||
if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
|
||||
if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(err) != testCase.s3Error {
|
||||
t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(err))
|
||||
if s3Error := isReqAuthenticated(ctx, testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
|
||||
if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
|
||||
t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %d)", i, testCase.s3Error, s3Error, toAPIErrorCode(ctx, err))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -413,8 +415,9 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
{Request: mustNewPresignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
{Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
}
|
||||
ctx := context.Background()
|
||||
for i, testCase := range testCases {
|
||||
if s3Error := checkAdminRequestAuthType(testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
|
||||
if s3Error := checkAdminRequestAuthType(ctx, testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
|
||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -123,7 +123,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
md5hex = getMD5Hash([]byte(textPartData))
|
||||
var partInfo PartInfo
|
||||
partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
|
||||
mustGetHashReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), opts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -204,7 +204,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -304,7 +304,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
for pb.Next() {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -344,7 +344,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetHashReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), metadata, ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
@@ -58,17 +59,19 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
|
||||
func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectsV2")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -78,14 +81,14 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
|
||||
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponse(w, errCode, r.URL)
|
||||
writeErrorResponse(w, errCode, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the query params before beginning to serve the request.
|
||||
// fetch-owner is not validated since it is a boolean
|
||||
if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listObjectsV2 := objectAPI.ListObjectsV2
|
||||
@@ -97,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -107,15 +110,16 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// Read the decompressed size from the meta.json.
|
||||
actualSize = listObjectsV2Info.Objects[i].GetActualSize()
|
||||
if actualSize < 0 {
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Set the info.Size to the actualSize.
|
||||
listObjectsV2Info.Objects[i].Size = actualSize
|
||||
} else if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
|
||||
listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
|
||||
listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -137,31 +141,37 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectsV1")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Extract all the litsObjectsV1 query params to their native values.
|
||||
prefix, marker, delimiter, maxKeys, _ := getListObjectsV1Args(r.URL.Query())
|
||||
prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
|
||||
// does not throw an error.
|
||||
if maxKeys < 0 {
|
||||
writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidMaxKeys, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
} // Validate all the query params before beginning to serve the request.
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listObjects := objectAPI.ListObjects
|
||||
@@ -173,7 +183,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -183,15 +193,16 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
// Read the decompressed size from the meta.json.
|
||||
actualSize = listObjectsInfo.Objects[i].GetActualSize()
|
||||
if actualSize < 0 {
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Set the info.Size to the actualSize.
|
||||
listObjectsInfo.Objects[i].Size = actualSize
|
||||
} else if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
|
||||
listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
|
||||
listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
+134
-87
@@ -89,17 +89,19 @@ func initFederatorBackend(objLayer ObjectLayer) {
|
||||
func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketLocation")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -108,7 +110,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
getBucketInfo = api.CacheAPI().GetBucketInfo
|
||||
}
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -137,36 +139,42 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListMultipartUploads")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _ := getBucketMultipartResources(r.URL.Query())
|
||||
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if maxUploads < 0 {
|
||||
writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidMaxUploads, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if keyMarker != "" {
|
||||
// Marker not common with prefix is not implemented.
|
||||
if !hasPrefix(keyMarker, prefix) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// generate response
|
||||
@@ -184,9 +192,11 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListBuckets")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listBuckets := objectAPI.ListBuckets
|
||||
@@ -196,7 +206,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// If etcd, dns federation configured list buckets from etcd.
|
||||
@@ -204,7 +214,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
if globalDNSConfig != nil {
|
||||
dnsBuckets, err := globalDNSConfig.List()
|
||||
if err != nil && err != dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
bucketSet := set.NewStringSet()
|
||||
@@ -223,7 +233,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
var err error
|
||||
bucketsInfo, err = listBuckets(ctx)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -240,12 +250,14 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteMultipleObjects")
|
||||
|
||||
defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -254,7 +266,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// In the event access is denied, a 200 response should still be returned
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if s3Error != ErrAccessDenied {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -262,14 +274,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Content-Length is required and should be non-zero
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Content-Md5 is requied should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if _, ok := r.Header["Content-Md5"]; !ok {
|
||||
writeErrorResponse(w, ErrMissingContentMD5, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentMD5, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -285,7 +297,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Read incoming body XML bytes.
|
||||
if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -293,7 +305,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
deleteObjects := &DeleteObjectsRequest{}
|
||||
if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedXML, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedXML, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -301,7 +313,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
if globalWORMEnabled {
|
||||
// Not required to check whether given objects exist or not, because
|
||||
// DeleteMultipleObject is always successful irrespective of object existence.
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -342,8 +354,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}
|
||||
// Error during delete should be collected separately.
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: errorCodeResponse[toAPIErrorCode(err)].Code,
|
||||
Message: errorCodeResponse[toAPIErrorCode(err)].Description,
|
||||
Code: errorCodeResponse[toAPIErrorCode(ctx, err)].Code,
|
||||
Message: errorCodeResponse[toAPIErrorCode(ctx, err)].Description,
|
||||
Key: object.ObjectName,
|
||||
})
|
||||
}
|
||||
@@ -370,10 +382,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
Object: ObjectInfo{
|
||||
Name: dobj.ObjectName,
|
||||
},
|
||||
ReqParams: extractReqParams(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: host,
|
||||
Port: port,
|
||||
ReqParams: extractReqParams(r),
|
||||
RespElements: extractRespElements(w),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: host,
|
||||
Port: port,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -384,9 +397,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -394,21 +409,21 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Parse incoming location constraint.
|
||||
location, s3Error := parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if location sent by the client is valid, reject
|
||||
// requests which do not follow valid region requirements.
|
||||
if !isValidLocation(location) {
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -417,12 +432,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
// Proceed to creating a bucket.
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if err = globalDNSConfig.Put(bucket); err != nil {
|
||||
objectAPI.DeleteBucket(ctx, bucket)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -432,18 +447,18 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
|
||||
}
|
||||
writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
|
||||
writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -460,28 +475,37 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PostPolicyBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if !api.EncryptionEnabled() && hasServerSideEncryptionHeader(r.Header) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
bucket := mux.Vars(r)["bucket"]
|
||||
|
||||
// Require Content-Length to be set in the request
|
||||
size := r.ContentLength
|
||||
if size < 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Make sure that the URL does not contain object name.
|
||||
if bucket != filepath.Clean(resource[1:]) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -490,7 +514,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
reader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -498,7 +522,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
form, err := reader.ReadForm(maxFormMemory)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -509,13 +533,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if file is provided, error out otherwise.
|
||||
if fileBody == nil {
|
||||
writeErrorResponse(w, ErrPOSTFileRequired, r.URL)
|
||||
writeErrorResponse(w, ErrPOSTFileRequired, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -537,7 +561,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if successRedirect != "" {
|
||||
redirectURL, err = url.Parse(successRedirect)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -545,58 +569,75 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// Verify policy signature.
|
||||
apiErr := doesPolicySignatureMatch(formValues)
|
||||
if apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure formValues adhere to policy restrictions.
|
||||
if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure that the object size is within expected range, also the file size
|
||||
// should not exceed the maximum single Put size (5 GiB)
|
||||
lengthRange := postPolicyForm.Conditions.ContentLengthRange
|
||||
if lengthRange.Valid {
|
||||
if fileSize < lengthRange.Min {
|
||||
writeErrorResponse(w, toAPIErrorCode(errDataTooSmall), r.URL)
|
||||
// Handle policy if it is set.
|
||||
if len(policyBytes) > 0 {
|
||||
postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
|
||||
writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
|
||||
// Make sure formValues adhere to policy restrictions.
|
||||
if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure that the object size is within expected range, also the file size
|
||||
// should not exceed the maximum single Put size (5 GiB)
|
||||
lengthRange := postPolicyForm.Conditions.ContentLengthRange
|
||||
if lengthRange.Valid {
|
||||
if fileSize < lengthRange.Min {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract metadata to be saved from received Form.
|
||||
metadata := make(map[string]string)
|
||||
err = extractMetadataFromMap(ctx, formValues, metadata)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
rawReader := hashReader
|
||||
pReader := NewPutObjReader(rawReader, nil, nil)
|
||||
var objectEncryptionKey []byte
|
||||
|
||||
// This request header needs to be set prior to setting ObjectOptions
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
|
||||
r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
}
|
||||
// get gateway encryption options
|
||||
var opts ObjectOptions
|
||||
opts, err = putEncryptionOpts(ctx, r, bucket, object, nil)
|
||||
if err != nil {
|
||||
writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
|
||||
return
|
||||
}
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
|
||||
var reader io.Reader
|
||||
@@ -604,27 +645,28 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if crypto.SSEC.IsRequested(formValues) {
|
||||
key, err = ParseSSECustomerHeader(formValues)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
reader, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
|
||||
reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
info := ObjectInfo{Size: fileSize}
|
||||
hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
|
||||
}
|
||||
}
|
||||
|
||||
objInfo, err := objectAPI.PutObject(ctx, bucket, object, hashReader, metadata, ObjectOptions{})
|
||||
objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, opts)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -640,13 +682,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
// Notify object created event.
|
||||
defer sendEvent(eventArgs{
|
||||
EventName: event.ObjectCreatedPost,
|
||||
BucketName: objInfo.Bucket,
|
||||
Object: objInfo,
|
||||
ReqParams: extractReqParams(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: host,
|
||||
Port: port,
|
||||
EventName: event.ObjectCreatedPost,
|
||||
BucketName: objInfo.Bucket,
|
||||
Object: objInfo,
|
||||
ReqParams: extractReqParams(r),
|
||||
RespElements: extractRespElements(w),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: host,
|
||||
Port: port,
|
||||
})
|
||||
|
||||
if successRedirect != "" {
|
||||
@@ -682,6 +725,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HeadBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
@@ -701,7 +746,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
getBucketInfo = api.CacheAPI().GetBucketInfo
|
||||
}
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
|
||||
writeErrorResponseHeadersOnly(w, toAPIErrorCode(ctx, err))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -712,17 +757,19 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -732,7 +779,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
// Attempt to delete bucket.
|
||||
if err := deleteBucket(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -744,7 +791,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
if err := globalDNSConfig.Delete(bucket); err != nil {
|
||||
// Deleting DNS entry failed, attempt to create the bucket again.
|
||||
objectAPI.MakeBucketWithLocation(ctx, bucket, "")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -625,7 +625,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
for i := 0; i < 10; i++ {
|
||||
objectName := "test-object-" + strconv.Itoa(i)
|
||||
// uploading the object.
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), nil, ObjectOptions{})
|
||||
// if object upload fails stop the test.
|
||||
if err != nil {
|
||||
t.Fatalf("Put Object %d: Error uploading object: <ERROR> %v", i, err)
|
||||
|
||||
@@ -44,28 +44,30 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
|
||||
func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketNotification")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucketName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -74,17 +76,21 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
if err != nil {
|
||||
// Ignore errNoSuchNotifications to comply with AWS S3.
|
||||
if err != errNoSuchNotifications {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
nConfig = &event.Config{}
|
||||
}
|
||||
|
||||
// If xml namespace is empty, set a default value before returning.
|
||||
if nConfig.XMLNS == "" {
|
||||
nConfig.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/"
|
||||
}
|
||||
|
||||
notificationBytes, err := xml.Marshal(nConfig)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -96,14 +102,16 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketNotification")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objectAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -111,19 +119,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucketName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucketNotification always needs a Content-Length.
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -132,15 +140,15 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
if err != nil {
|
||||
apiErr := ErrMalformedXML
|
||||
if event.IsEventError(err) {
|
||||
apiErr = toAPIErrorCode(err)
|
||||
apiErr = toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = saveNotificationConfig(objectAPI, bucketName, config); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -156,22 +164,28 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListenBucketNotification")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListenBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
// Validate if bucket exists.
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if !objAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objAPI.IsListenBucketSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -179,11 +193,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
var prefix string
|
||||
if len(values["prefix"]) > 1 {
|
||||
writeErrorResponse(w, ErrFilterNamePrefix, r.URL)
|
||||
writeErrorResponse(w, ErrFilterNamePrefix, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
if len(values["prefix"]) == 1 {
|
||||
if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -192,11 +206,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
var suffix string
|
||||
if len(values["suffix"]) > 1 {
|
||||
writeErrorResponse(w, ErrFilterNameSuffix, r.URL)
|
||||
writeErrorResponse(w, ErrFilterNameSuffix, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
if len(values["suffix"]) == 1 {
|
||||
if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -209,7 +223,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
for _, s := range values["events"] {
|
||||
eventName, err := event.ParseName(s)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -217,19 +231,19 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
host, err := xnet.ParseHost(r.RemoteAddr)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
target, err := target.NewHTTPClientTarget(*host, w)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -237,8 +251,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
|
||||
@@ -246,14 +259,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -263,8 +275,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,9 +40,11 @@ const (
|
||||
func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -50,43 +52,43 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is missing.
|
||||
// PutBucketPolicy always needs Content-Length.
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
if r.ContentLength > maxBucketPolicySize {
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL)
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Version in policy must not be empty
|
||||
if bucketPolicy.Version == "" {
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -101,9 +103,11 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -111,18 +115,18 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -137,9 +141,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -147,27 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Read bucket access policy.
|
||||
bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
policyData, err := json.Marshal(bucketPolicy)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+10
-10
@@ -71,9 +71,13 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
|
||||
// Get all CA file names.
|
||||
var caFiles []string
|
||||
fis, err := readDir(certsCAsDir)
|
||||
if err != nil {
|
||||
if err != nil && err != errFileNotFound {
|
||||
return nil, err
|
||||
}
|
||||
// Return success if CA's directory is missing.
|
||||
if err == errFileNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
for _, fi := range fis {
|
||||
// Skip all directories.
|
||||
if hasSuffix(fi, slashSeparator) {
|
||||
@@ -150,24 +154,20 @@ func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
|
||||
return cert, nil
|
||||
}
|
||||
|
||||
func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, c *certs.Certs, secureConn bool, err error) {
|
||||
func getTLSConfig() (x509Certs []*x509.Certificate, c *certs.Certs, secureConn bool, err error) {
|
||||
if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
|
||||
return nil, nil, nil, false, nil
|
||||
return nil, nil, false, nil
|
||||
}
|
||||
|
||||
if x509Certs, err = parsePublicCertFile(getPublicCertFile()); err != nil {
|
||||
return nil, nil, nil, false, err
|
||||
return nil, nil, false, err
|
||||
}
|
||||
|
||||
c, err = certs.New(getPublicCertFile(), getPrivateKeyFile(), loadX509KeyPair)
|
||||
if err != nil {
|
||||
return nil, nil, nil, false, err
|
||||
}
|
||||
|
||||
if rootCAs, err = getRootCAs(getCADir()); err != nil {
|
||||
return nil, nil, nil, false, err
|
||||
return nil, nil, false, err
|
||||
}
|
||||
|
||||
secureConn = true
|
||||
return x509Certs, rootCAs, c, secureConn, nil
|
||||
return x509Certs, c, secureConn, nil
|
||||
}
|
||||
|
||||
+2
-1
@@ -223,7 +223,8 @@ func TestGetRootCAs(t *testing.T) {
|
||||
certCAsDir string
|
||||
expectedErr error
|
||||
}{
|
||||
{"nonexistent-dir", errFileNotFound},
|
||||
// ignores non-existent directories.
|
||||
{"nonexistent-dir", nil},
|
||||
// Ignores directories.
|
||||
{dir1, nil},
|
||||
// Ignore empty directory.
|
||||
|
||||
+151
-63
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"os"
|
||||
@@ -28,12 +29,13 @@ import (
|
||||
etcd "github.com/coreos/etcd/clientv3"
|
||||
dns2 "github.com/miekg/dns"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/logger/target/console"
|
||||
"github.com/minio/minio/cmd/logger/target/http"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
// Check for updates and print a notification message
|
||||
@@ -50,50 +52,101 @@ func checkUpdate(mode string) {
|
||||
|
||||
// Load logger targets based on user's configuration
|
||||
func loadLoggers() {
|
||||
if globalServerConfig.Logger.Console.Enabled {
|
||||
// Enable console logging
|
||||
logger.AddTarget(logger.NewConsole())
|
||||
auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
|
||||
if ok {
|
||||
// Enable audit HTTP logging through ENV.
|
||||
logger.AddAuditTarget(http.New(auditEndpoint, NewCustomHTTPTransport()))
|
||||
}
|
||||
for _, l := range globalServerConfig.Logger.HTTP {
|
||||
if l.Enabled {
|
||||
// Enable http logging
|
||||
logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
|
||||
|
||||
loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
|
||||
if ok {
|
||||
// Enable HTTP logging through ENV.
|
||||
logger.AddTarget(http.New(loggerEndpoint, NewCustomHTTPTransport()))
|
||||
} else {
|
||||
for _, l := range globalServerConfig.Logger.HTTP {
|
||||
if l.Enabled {
|
||||
// Enable http logging
|
||||
logger.AddTarget(http.New(l.Endpoint, NewCustomHTTPTransport()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if globalServerConfig.Logger.Console.Enabled {
|
||||
// Enable console logging
|
||||
logger.AddTarget(console.New())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) *ConfigDir {
|
||||
var dir string
|
||||
|
||||
switch {
|
||||
case ctx.IsSet(option):
|
||||
dir = ctx.String(option)
|
||||
case ctx.GlobalIsSet(option):
|
||||
dir = ctx.GlobalString(option)
|
||||
// cli package does not expose parent's option option. Below code is workaround.
|
||||
if dir == "" || dir == getDefaultDir() {
|
||||
if ctx.Parent().GlobalIsSet(option) {
|
||||
dir = ctx.Parent().GlobalString(option)
|
||||
}
|
||||
}
|
||||
default:
|
||||
// Neither local nor global option is provided. In this case, try to use
|
||||
// default directory.
|
||||
dir = getDefaultDir()
|
||||
if dir == "" {
|
||||
logger.FatalIf(errInvalidArgument, "%s option must be provided", option)
|
||||
}
|
||||
}
|
||||
|
||||
if dir == "" {
|
||||
logger.FatalIf(errors.New("empty directory"), "%s directory cannot be empty", option)
|
||||
}
|
||||
|
||||
// Disallow relative paths, figure out absolute paths.
|
||||
dirAbs, err := filepath.Abs(dir)
|
||||
logger.FatalIf(err, "Unable to fetch absolute path for %s=%s", option, dir)
|
||||
|
||||
logger.FatalIf(mkdirAllIgnorePerm(dirAbs), "Unable to create directory specified %s=%s", option, dir)
|
||||
|
||||
return &ConfigDir{path: dirAbs}
|
||||
}
|
||||
|
||||
func handleCommonCmdArgs(ctx *cli.Context) {
|
||||
|
||||
var configDir string
|
||||
|
||||
switch {
|
||||
case ctx.IsSet("config-dir"):
|
||||
configDir = ctx.String("config-dir")
|
||||
case ctx.GlobalIsSet("config-dir"):
|
||||
configDir = ctx.GlobalString("config-dir")
|
||||
// cli package does not expose parent's "config-dir" option. Below code is workaround.
|
||||
if configDir == "" || configDir == getConfigDir() {
|
||||
if ctx.Parent().GlobalIsSet("config-dir") {
|
||||
configDir = ctx.Parent().GlobalString("config-dir")
|
||||
}
|
||||
}
|
||||
default:
|
||||
// Neither local nor global config-dir option is provided. In this case, try to use
|
||||
// default config directory.
|
||||
configDir = getConfigDir()
|
||||
if configDir == "" {
|
||||
logger.FatalIf(errors.New("missing option"), "config-dir option must be provided")
|
||||
}
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if globalCLIContext.JSON {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
if configDir == "" {
|
||||
logger.FatalIf(errors.New("empty directory"), "Configuration directory cannot be empty")
|
||||
// Get quiet flag from command line argument.
|
||||
globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if globalCLIContext.Quiet {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Disallow relative paths, figure out absolute paths.
|
||||
configDirAbs, err := filepath.Abs(configDir)
|
||||
logger.FatalIf(err, "Unable to fetch absolute path for config directory %s", configDir)
|
||||
setConfigDir(configDirAbs)
|
||||
// Get anonymous flag from command line argument.
|
||||
globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
|
||||
if globalCLIContext.Anonymous {
|
||||
logger.EnableAnonymous()
|
||||
}
|
||||
|
||||
// Fetch address option
|
||||
globalCLIContext.Addr = ctx.GlobalString("address")
|
||||
if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+globalMinioDefaultPort {
|
||||
globalCLIContext.Addr = ctx.String("address")
|
||||
}
|
||||
|
||||
// Set all config, certs and CAs directories.
|
||||
globalConfigDir = newConfigDirFromCtx(ctx, "config-dir", defaultConfigDir.Get)
|
||||
globalCertsDir = newConfigDirFromCtx(ctx, "certs-dir", defaultCertsDir.Get)
|
||||
globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
|
||||
|
||||
logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
|
||||
}
|
||||
|
||||
// Parses the given compression exclude list `extensions` or `content-types`.
|
||||
@@ -122,6 +175,7 @@ func handleCommonEnvVars() {
|
||||
if err != nil {
|
||||
logger.Fatal(uiErrInvalidCredentials(err), "Unable to validate credentials inherited from the shell environment")
|
||||
}
|
||||
cred.Expiration = timeSentinel
|
||||
|
||||
// credential Envs are set globally.
|
||||
globalIsEnvCreds = true
|
||||
@@ -131,7 +185,7 @@ func handleCommonEnvVars() {
|
||||
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
|
||||
browserFlag, err := ParseBoolFlag(browser)
|
||||
if err != nil {
|
||||
logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER environment variable")
|
||||
logger.Fatal(uiErrInvalidBrowserValue(nil).Msg("Unknown value `%s`", browser), "Invalid MINIO_BROWSER value in environment variable")
|
||||
}
|
||||
|
||||
// browser Envs are set globally, this does not represent
|
||||
@@ -150,33 +204,81 @@ func handleCommonEnvVars() {
|
||||
etcdEndpointsEnv, ok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
|
||||
if ok {
|
||||
etcdEndpoints := strings.Split(etcdEndpointsEnv, ",")
|
||||
|
||||
var etcdSecure bool
|
||||
for _, endpoint := range etcdEndpoints {
|
||||
u, err := xnet.ParseURL(endpoint)
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
|
||||
}
|
||||
// If one of the endpoint is https, we will use https directly.
|
||||
etcdSecure = etcdSecure || u.Scheme == "https"
|
||||
}
|
||||
|
||||
var err error
|
||||
globalEtcdClient, err = etcd.New(etcd.Config{
|
||||
Endpoints: etcdEndpoints,
|
||||
DialTimeout: defaultDialTimeout,
|
||||
DialKeepAliveTime: defaultDialKeepAlive,
|
||||
})
|
||||
if etcdSecure {
|
||||
// This is only to support client side certificate authentication
|
||||
// https://coreos.com/etcd/docs/latest/op-guide/security.html
|
||||
etcdClientCertFile, ok1 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT")
|
||||
etcdClientCertKey, ok2 := os.LookupEnv("MINIO_ETCD_CLIENT_CERT_KEY")
|
||||
var getClientCertificate func(*tls.CertificateRequestInfo) (*tls.Certificate, error)
|
||||
if ok1 && ok2 {
|
||||
getClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
cert, terr := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey)
|
||||
return &cert, terr
|
||||
}
|
||||
}
|
||||
globalEtcdClient, err = etcd.New(etcd.Config{
|
||||
Endpoints: etcdEndpoints,
|
||||
DialTimeout: defaultDialTimeout,
|
||||
DialKeepAliveTime: defaultDialKeepAlive,
|
||||
TLS: &tls.Config{
|
||||
RootCAs: globalRootCAs,
|
||||
GetClientCertificate: getClientCertificate,
|
||||
},
|
||||
})
|
||||
} else {
|
||||
globalEtcdClient, err = etcd.New(etcd.Config{
|
||||
Endpoints: etcdEndpoints,
|
||||
DialTimeout: defaultDialTimeout,
|
||||
DialKeepAliveTime: defaultDialKeepAlive,
|
||||
})
|
||||
}
|
||||
logger.FatalIf(err, "Unable to initialize etcd with %s", etcdEndpoints)
|
||||
}
|
||||
|
||||
globalDomainName, globalIsEnvDomainName = os.LookupEnv("MINIO_DOMAIN")
|
||||
if globalDomainName != "" {
|
||||
if _, ok = dns2.IsDomainName(globalDomainName); !ok {
|
||||
logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN environment variable")
|
||||
logger.Fatal(uiErrInvalidDomainValue(nil).Msg("Unknown value `%s`", globalDomainName), "Invalid MINIO_DOMAIN value in environment variable")
|
||||
}
|
||||
}
|
||||
|
||||
minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if ok {
|
||||
minioEndpoints := strings.Split(minioEndpointsEnv, ",")
|
||||
globalDomainIPs = set.NewStringSet()
|
||||
for i, ip := range minioEndpoints {
|
||||
if net.ParseIP(ip) == nil {
|
||||
logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
|
||||
var domainIPs = set.NewStringSet()
|
||||
for _, endpoint := range minioEndpoints {
|
||||
if net.ParseIP(endpoint) == nil {
|
||||
// Checking if the IP is a DNS entry.
|
||||
addrs, err := net.LookupHost(endpoint)
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to initialize Minio server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
domainIPs.Add(addr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
globalDomainIPs.Add(ip)
|
||||
domainIPs.Add(endpoint)
|
||||
}
|
||||
updateDomainIPs(domainIPs)
|
||||
} else {
|
||||
// Add found interfaces IP address to global domain IPS,
|
||||
// loopback addresses will be naturally dropped.
|
||||
updateDomainIPs(localIP4)
|
||||
}
|
||||
|
||||
if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
var err error
|
||||
globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
|
||||
@@ -257,7 +359,7 @@ func handleCommonEnvVars() {
|
||||
if worm := os.Getenv("MINIO_WORM"); worm != "" {
|
||||
wormFlag, err := ParseBoolFlag(worm)
|
||||
if err != nil {
|
||||
logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Unable to validate MINIO_WORM environment variable")
|
||||
logger.Fatal(uiErrInvalidWormValue(nil).Msg("Unknown value `%s`", worm), "Invalid MINIO_WORM value in environment variable")
|
||||
}
|
||||
|
||||
// worm Envs are set globally, this does not represent
|
||||
@@ -266,20 +368,6 @@ func handleCommonEnvVars() {
|
||||
globalWORMEnabled = bool(wormFlag)
|
||||
}
|
||||
|
||||
kmsConf, err := crypto.NewVaultConfig()
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize hashicorp vault")
|
||||
}
|
||||
if kmsConf.Vault.Endpoint != "" {
|
||||
kms, err := crypto.NewVault(kmsConf)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize KMS")
|
||||
}
|
||||
globalKMS = kms
|
||||
globalKMSKeyID = kmsConf.Vault.Key.Name
|
||||
globalKMSConfig = kmsConf
|
||||
}
|
||||
|
||||
if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
|
||||
globalIsCompressionEnabled = strings.EqualFold(compress, "true")
|
||||
}
|
||||
|
||||
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
etcd "github.com/coreos/etcd/clientv3"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
var errConfigNotFound = errors.New("config file not found")
|
||||
|
||||
func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
|
||||
var buffer bytes.Buffer
|
||||
// Read entire content by setting size to -1
|
||||
if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
|
||||
// Treat object not found as config not found.
|
||||
if isErrObjectNotFound(err) {
|
||||
return nil, errConfigNotFound
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Return config not found on empty content.
|
||||
if buffer.Len() == 0 {
|
||||
return nil, errConfigNotFound
|
||||
}
|
||||
|
||||
return buffer.Bytes(), nil
|
||||
}
|
||||
|
||||
func deleteConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
|
||||
_, err := client.Delete(ctx, configFile)
|
||||
return err
|
||||
}
|
||||
|
||||
func deleteConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
|
||||
return objAPI.DeleteObject(ctx, minioMetaBucket, configFile)
|
||||
}
|
||||
|
||||
func saveConfigEtcd(ctx context.Context, client *etcd.Client, configFile string, data []byte) error {
|
||||
_, err := client.Put(ctx, configFile, string(data))
|
||||
return err
|
||||
}
|
||||
|
||||
func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data []byte) error {
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), nil, ObjectOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) ([]byte, error) {
|
||||
resp, err := client.Get(ctx, configFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Count == 0 {
|
||||
return nil, errConfigNotFound
|
||||
}
|
||||
for _, ev := range resp.Kvs {
|
||||
if string(ev.Key) == configFile {
|
||||
return ev.Value, nil
|
||||
}
|
||||
}
|
||||
return nil, errConfigNotFound
|
||||
}
|
||||
|
||||
// watchConfig - watches for changes on `configFile` on etcd and loads them.
|
||||
func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
|
||||
if globalEtcdClient != nil {
|
||||
for watchResp := range globalEtcdClient.Watch(context.Background(), configFile) {
|
||||
for _, event := range watchResp.Events {
|
||||
if event.IsModify() || event.IsCreate() {
|
||||
loadCfgFn(objAPI)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkConfigEtcd(ctx context.Context, client *etcd.Client, configFile string) error {
|
||||
resp, err := globalEtcdClient.Get(ctx, configFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Count == 0 {
|
||||
return errConfigNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error {
|
||||
if globalEtcdClient != nil {
|
||||
return checkConfigEtcd(ctx, globalEtcdClient, configFile)
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
|
||||
// Treat object not found as config not found.
|
||||
if isErrObjectNotFound(err) {
|
||||
return errConfigNotFound
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+100
-22
@@ -20,15 +20,18 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/event/target"
|
||||
"github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/iam/validator"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
// Steps to move from version N to version N+1
|
||||
@@ -40,9 +43,9 @@ import (
|
||||
// 6. Make changes in config-current_test.go for any test change
|
||||
|
||||
// Config version
|
||||
const serverConfigVersion = "30"
|
||||
const serverConfigVersion = "33"
|
||||
|
||||
type serverConfig = serverConfigV30
|
||||
type serverConfig = serverConfigV33
|
||||
|
||||
var (
|
||||
// globalServerConfig server config.
|
||||
@@ -173,55 +176,61 @@ func (s *serverConfig) Validate() error {
|
||||
// Worm, Cache and StorageClass values are already validated during json unmarshal
|
||||
for _, v := range s.Notify.AMQP {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("amqp: %s", err.Error())
|
||||
return fmt.Errorf("amqp: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Elasticsearch {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("elasticsearch: %s", err.Error())
|
||||
return fmt.Errorf("elasticsearch: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Kafka {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("kafka: %s", err.Error())
|
||||
return fmt.Errorf("kafka: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.MQTT {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("mqtt: %s", err.Error())
|
||||
return fmt.Errorf("mqtt: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.MySQL {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("mysql: %s", err.Error())
|
||||
return fmt.Errorf("mysql: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.NATS {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("nats: %s", err.Error())
|
||||
return fmt.Errorf("nats: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.NSQ {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("nsq: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.PostgreSQL {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("postgreSQL: %s", err.Error())
|
||||
return fmt.Errorf("postgreSQL: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Redis {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("redis: %s", err.Error())
|
||||
return fmt.Errorf("redis: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, v := range s.Notify.Webhook {
|
||||
if err := v.Validate(); err != nil {
|
||||
return fmt.Errorf("webhook: %s", err.Error())
|
||||
return fmt.Errorf("webhook: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,13 +271,27 @@ func (s *serverConfig) loadFromEnvs() {
|
||||
s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
|
||||
}
|
||||
|
||||
if globalKMS != nil {
|
||||
s.KMS = globalKMSConfig
|
||||
if err := Environment.LookupKMSConfig(s.KMS); err != nil {
|
||||
logger.FatalIf(err, "Unable to setup the KMS")
|
||||
}
|
||||
|
||||
if globalIsEnvCompression {
|
||||
s.SetCompressionConfig(globalCompressExtensions, globalCompressMimeTypes)
|
||||
}
|
||||
|
||||
if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
|
||||
if u, err := xnet.ParseURL(jwksURL); err == nil {
|
||||
s.OpenID.JWKS.URL = u
|
||||
s.OpenID.JWKS.PopulatePublicKey()
|
||||
}
|
||||
}
|
||||
|
||||
if opaURL, ok := os.LookupEnv("MINIO_IAM_OPA_URL"); ok {
|
||||
if u, err := xnet.ParseURL(opaURL); err == nil {
|
||||
s.Policy.OPA.URL = u
|
||||
s.Policy.OPA.AuthToken = os.Getenv("MINIO_IAM_OPA_AUTHTOKEN")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNotificationTargets tries to establish connections to all notification
|
||||
@@ -341,6 +364,17 @@ func (s *serverConfig) TestNotificationTargets() error {
|
||||
t.Close()
|
||||
}
|
||||
|
||||
for k, v := range s.Notify.NSQ {
|
||||
if !v.Enable {
|
||||
continue
|
||||
}
|
||||
t, err := target.NewNSQTarget(k, v)
|
||||
if err != nil {
|
||||
return fmt.Errorf("nsq(%s): %s", k, err.Error())
|
||||
}
|
||||
t.Close()
|
||||
}
|
||||
|
||||
for k, v := range s.Notify.PostgreSQL {
|
||||
if !v.Enable {
|
||||
continue
|
||||
@@ -388,6 +422,8 @@ func (s *serverConfig) ConfigDiff(t *serverConfig) string {
|
||||
return "AMQP Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
|
||||
return "NATS Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.NSQ, t.Notify.NSQ):
|
||||
return "NSQ Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.Elasticsearch, t.Notify.Elasticsearch):
|
||||
return "ElasticSearch Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
|
||||
@@ -453,6 +489,8 @@ func newServerConfig() *serverConfig {
|
||||
srvCfg.Notify.Redis["1"] = target.RedisArgs{}
|
||||
srvCfg.Notify.NATS = make(map[string]target.NATSArgs)
|
||||
srvCfg.Notify.NATS["1"] = target.NATSArgs{}
|
||||
srvCfg.Notify.NSQ = make(map[string]target.NSQArgs)
|
||||
srvCfg.Notify.NSQ["1"] = target.NSQArgs{}
|
||||
srvCfg.Notify.PostgreSQL = make(map[string]target.PostgreSQLArgs)
|
||||
srvCfg.Notify.PostgreSQL["1"] = target.PostgreSQLArgs{}
|
||||
srvCfg.Notify.MySQL = make(map[string]target.MySQLArgs)
|
||||
@@ -496,24 +534,36 @@ func (s *serverConfig) loadToCachedConfigs() {
|
||||
globalCacheExpiry = cacheConf.Expiry
|
||||
globalCacheMaxUse = cacheConf.MaxUse
|
||||
}
|
||||
if globalKMS == nil {
|
||||
globalKMSConfig = s.KMS
|
||||
if kms, err := crypto.NewVault(globalKMSConfig); err == nil {
|
||||
globalKMS = kms
|
||||
globalKMSKeyID = globalKMSConfig.Vault.Key.Name
|
||||
}
|
||||
if err := Environment.LookupKMSConfig(s.KMS); err != nil {
|
||||
logger.FatalIf(err, "Unable to setup the KMS")
|
||||
}
|
||||
|
||||
if !globalIsCompressionEnabled {
|
||||
compressionConf := s.GetCompressionConfig()
|
||||
globalCompressExtensions = compressionConf.Extensions
|
||||
globalCompressMimeTypes = compressionConf.MimeTypes
|
||||
globalIsCompressionEnabled = compressionConf.Enabled
|
||||
}
|
||||
|
||||
if globalIAMValidators == nil {
|
||||
globalIAMValidators = getAuthValidators(s)
|
||||
}
|
||||
|
||||
if globalPolicyOPA == nil {
|
||||
if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
|
||||
globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
|
||||
URL: s.Policy.OPA.URL,
|
||||
AuthToken: s.Policy.OPA.AuthToken,
|
||||
Transport: NewCustomHTTPTransport(),
|
||||
CloseRespFn: CloseResponse,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// newConfig - initialize a new server config, saves env parameters if
|
||||
// newSrvConfig - initialize a new server config, saves env parameters if
|
||||
// found, otherwise use default parameters
|
||||
func newConfig(objAPI ObjectLayer) error {
|
||||
func newSrvConfig(objAPI ObjectLayer) error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
@@ -564,6 +614,20 @@ func loadConfig(objAPI ObjectLayer) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// getAuthValidators - returns ValidatorList which contains
|
||||
// enabled providers in server config.
|
||||
// A new authentication provider is added like below
|
||||
// * Add a new provider in pkg/iam/validator package.
|
||||
func getAuthValidators(config *serverConfig) *validator.Validators {
|
||||
validators := validator.NewValidators()
|
||||
|
||||
if config.OpenID.JWKS.URL != nil {
|
||||
validators.Add(validator.NewJWT(config.OpenID.JWKS))
|
||||
}
|
||||
|
||||
return validators
|
||||
}
|
||||
|
||||
// getNotificationTargets - returns TargetList which contains enabled targets in serverConfig.
|
||||
// A new notification target is added like below
|
||||
// * Add a new target in pkg/event/target package.
|
||||
@@ -660,6 +724,20 @@ func getNotificationTargets(config *serverConfig) *event.TargetList {
|
||||
}
|
||||
}
|
||||
|
||||
for id, args := range config.Notify.NSQ {
|
||||
if args.Enable {
|
||||
newTarget, err := target.NewNSQTarget(id, args)
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
continue
|
||||
}
|
||||
if err = targetList.Add(newTarget); err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for id, args := range config.Notify.PostgreSQL {
|
||||
if args.Enable {
|
||||
newTarget, err := target.NewPostgreSQLTarget(id, args)
|
||||
|
||||
+28
-11
@@ -188,7 +188,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
|
||||
|
||||
// Test 12 - Test NATS
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
|
||||
// Test 13 - Test ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
|
||||
@@ -234,10 +234,13 @@ func TestValidateConfig(t *testing.T) {
|
||||
|
||||
// Test 27 - Test MQTT
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "", "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
|
||||
|
||||
// Test 28 - Test NSQ
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
if err = saveConfig(objLayer, configPath, []byte(testCase.configData)); err != nil {
|
||||
if err = saveConfig(context.Background(), objLayer, configPath, []byte(testCase.configData)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = getValidConfig(objLayer)
|
||||
@@ -260,8 +263,16 @@ func TestConfigDiff(t *testing.T) {
|
||||
{&serverConfig{}, nil, "Given configuration is empty"},
|
||||
// 2
|
||||
{
|
||||
&serverConfig{Credential: auth.Credentials{"u1", "p1"}},
|
||||
&serverConfig{Credential: auth.Credentials{"u1", "p2"}},
|
||||
&serverConfig{Credential: auth.Credentials{
|
||||
AccessKey: "u1",
|
||||
SecretKey: "p1",
|
||||
Expiration: timeSentinel,
|
||||
}},
|
||||
&serverConfig{Credential: auth.Credentials{
|
||||
AccessKey: "u1",
|
||||
SecretKey: "p2",
|
||||
Expiration: timeSentinel,
|
||||
}},
|
||||
"Credential configuration differs",
|
||||
},
|
||||
// 3
|
||||
@@ -285,48 +296,54 @@ func TestConfigDiff(t *testing.T) {
|
||||
"NATS Notification configuration differs",
|
||||
},
|
||||
// 7
|
||||
{
|
||||
&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{NSQ: map[string]target.NSQArgs{"1": {Enable: false}}}},
|
||||
"NSQ Notification configuration differs",
|
||||
},
|
||||
// 8
|
||||
{
|
||||
&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{Elasticsearch: map[string]target.ElasticsearchArgs{"1": {Enable: false}}}},
|
||||
"ElasticSearch Notification configuration differs",
|
||||
},
|
||||
// 8
|
||||
// 9
|
||||
{
|
||||
&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{Redis: map[string]target.RedisArgs{"1": {Enable: false}}}},
|
||||
"Redis Notification configuration differs",
|
||||
},
|
||||
// 9
|
||||
// 10
|
||||
{
|
||||
&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{PostgreSQL: map[string]target.PostgreSQLArgs{"1": {Enable: false}}}},
|
||||
"PostgreSQL Notification configuration differs",
|
||||
},
|
||||
// 10
|
||||
// 11
|
||||
{
|
||||
&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{Kafka: map[string]target.KafkaArgs{"1": {Enable: false}}}},
|
||||
"Kafka Notification configuration differs",
|
||||
},
|
||||
// 11
|
||||
// 12
|
||||
{
|
||||
&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{Webhook: map[string]target.WebhookArgs{"1": {Enable: false}}}},
|
||||
"Webhook Notification configuration differs",
|
||||
},
|
||||
// 12
|
||||
// 13
|
||||
{
|
||||
&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{MySQL: map[string]target.MySQLArgs{"1": {Enable: false}}}},
|
||||
"MySQL Notification configuration differs",
|
||||
},
|
||||
// 13
|
||||
// 14
|
||||
{
|
||||
&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{MQTT: map[string]target.MQTTArgs{"1": {Enable: false}}}},
|
||||
"MQTT Notification configuration differs",
|
||||
},
|
||||
// 14
|
||||
// 15
|
||||
{
|
||||
&serverConfig{Logger: loggerConfig{
|
||||
Console: loggerConsole{Enabled: true},
|
||||
|
||||
+37
-62
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
homedir "github.com/mitchellh/go-homedir"
|
||||
)
|
||||
@@ -41,55 +40,9 @@ const (
|
||||
privateKeyFile = "private.key"
|
||||
)
|
||||
|
||||
// ConfigDir - configuration directory with locking.
|
||||
// ConfigDir - points to a user set directory.
|
||||
type ConfigDir struct {
|
||||
sync.Mutex
|
||||
dir string
|
||||
}
|
||||
|
||||
// Set - saves given directory as configuration directory.
|
||||
func (config *ConfigDir) Set(dir string) {
|
||||
config.Lock()
|
||||
defer config.Unlock()
|
||||
|
||||
config.dir = dir
|
||||
}
|
||||
|
||||
// Get - returns current configuration directory.
|
||||
func (config *ConfigDir) Get() string {
|
||||
config.Lock()
|
||||
defer config.Unlock()
|
||||
|
||||
return config.dir
|
||||
}
|
||||
|
||||
func (config *ConfigDir) getCertsDir() string {
|
||||
return filepath.Join(config.Get(), certsDir)
|
||||
}
|
||||
|
||||
// GetCADir - returns certificate CA directory.
|
||||
func (config *ConfigDir) GetCADir() string {
|
||||
return filepath.Join(config.getCertsDir(), certsCADir)
|
||||
}
|
||||
|
||||
// Create - creates configuration directory tree.
|
||||
func (config *ConfigDir) Create() error {
|
||||
return os.MkdirAll(config.GetCADir(), 0700)
|
||||
}
|
||||
|
||||
// GetMinioConfigFile - returns absolute path of config.json file.
|
||||
func (config *ConfigDir) GetMinioConfigFile() string {
|
||||
return filepath.Join(config.Get(), minioConfigFile)
|
||||
}
|
||||
|
||||
// GetPublicCertFile - returns absolute path of public.crt file.
|
||||
func (config *ConfigDir) GetPublicCertFile() string {
|
||||
return filepath.Join(config.getCertsDir(), publicCertFile)
|
||||
}
|
||||
|
||||
// GetPrivateKeyFile - returns absolute path of private.key file.
|
||||
func (config *ConfigDir) GetPrivateKeyFile() string {
|
||||
return filepath.Join(config.getCertsDir(), privateKeyFile)
|
||||
path string
|
||||
}
|
||||
|
||||
func getDefaultConfigDir() string {
|
||||
@@ -101,32 +54,54 @@ func getDefaultConfigDir() string {
|
||||
return filepath.Join(homeDir, defaultMinioConfigDir)
|
||||
}
|
||||
|
||||
var configDir = &ConfigDir{dir: getDefaultConfigDir()}
|
||||
|
||||
func setConfigDir(dir string) {
|
||||
configDir.Set(dir)
|
||||
func getDefaultCertsDir() string {
|
||||
return filepath.Join(getDefaultConfigDir(), certsDir)
|
||||
}
|
||||
|
||||
func getConfigDir() string {
|
||||
return configDir.Get()
|
||||
func getDefaultCertsCADir() string {
|
||||
return filepath.Join(getDefaultCertsDir(), certsCADir)
|
||||
}
|
||||
|
||||
func getCADir() string {
|
||||
return configDir.GetCADir()
|
||||
var (
|
||||
// Default config, certs and CA directories.
|
||||
defaultConfigDir = &ConfigDir{path: getDefaultConfigDir()}
|
||||
defaultCertsDir = &ConfigDir{path: getDefaultCertsDir()}
|
||||
defaultCertsCADir = &ConfigDir{path: getDefaultCertsCADir()}
|
||||
|
||||
// Points to current configuration directory -- deprecated, to be removed in future.
|
||||
globalConfigDir = defaultConfigDir
|
||||
// Points to current certs directory set by user with --certs-dir
|
||||
globalCertsDir = defaultCertsDir
|
||||
// Points to relative path to certs directory and is <value-of-certs-dir>/CAs
|
||||
globalCertsCADir = defaultCertsCADir
|
||||
)
|
||||
|
||||
// Get - returns current directory.
|
||||
func (dir *ConfigDir) Get() string {
|
||||
return dir.path
|
||||
}
|
||||
|
||||
func createConfigDir() error {
|
||||
return configDir.Create()
|
||||
// Attempts to create all directories, ignores any permission denied errors.
|
||||
func mkdirAllIgnorePerm(path string) error {
|
||||
err := os.MkdirAll(path, 0700)
|
||||
if err != nil {
|
||||
// It is possible in kubernetes like deployments this directory
|
||||
// is already mounted and is not writable, ignore any write errors.
|
||||
if os.IsPermission(err) {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func getConfigFile() string {
|
||||
return configDir.GetMinioConfigFile()
|
||||
return filepath.Join(globalConfigDir.Get(), minioConfigFile)
|
||||
}
|
||||
|
||||
func getPublicCertFile() string {
|
||||
return configDir.GetPublicCertFile()
|
||||
return filepath.Join(globalCertsDir.Get(), publicCertFile)
|
||||
}
|
||||
|
||||
func getPrivateKeyFile() string {
|
||||
return configDir.GetPrivateKeyFile()
|
||||
return filepath.Join(globalCertsDir.Get(), privateKeyFile)
|
||||
}
|
||||
|
||||
+268
-41
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
@@ -29,6 +30,8 @@ import (
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/event/target"
|
||||
"github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/iam/validator"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
)
|
||||
@@ -228,7 +231,7 @@ func migrateConfig() error {
|
||||
|
||||
// Version '1' is not supported anymore and deprecated, safe to delete.
|
||||
func purgeV1() error {
|
||||
configFile := filepath.Join(getConfigDir(), "fsUsers.json")
|
||||
configFile := filepath.Join(globalConfigDir.Get(), "fsUsers.json")
|
||||
|
||||
cv1 := &configV1{}
|
||||
_, err := Load(configFile, cv1)
|
||||
@@ -916,7 +919,7 @@ func migrateV12ToV13() error {
|
||||
// Copy over fields from V12 into V13 config struct
|
||||
srvConfig := &serverConfigV13{
|
||||
Logger: &loggerV7{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "13"
|
||||
srvConfig.Credential = cv12.Credential
|
||||
@@ -996,7 +999,7 @@ func migrateV13ToV14() error {
|
||||
// Copy over fields from V13 into V14 config struct
|
||||
srvConfig := &serverConfigV14{
|
||||
Logger: &loggerV7{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "14"
|
||||
srvConfig.Credential = cv13.Credential
|
||||
@@ -1081,7 +1084,7 @@ func migrateV14ToV15() error {
|
||||
// Copy over fields from V14 into V15 config struct
|
||||
srvConfig := &serverConfigV15{
|
||||
Logger: &loggerV7{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "15"
|
||||
srvConfig.Credential = cv14.Credential
|
||||
@@ -1171,7 +1174,7 @@ func migrateV15ToV16() error {
|
||||
// Copy over fields from V15 into V16 config struct
|
||||
srvConfig := &serverConfigV16{
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "16"
|
||||
srvConfig.Credential = cv15.Credential
|
||||
@@ -1261,7 +1264,7 @@ func migrateV16ToV17() error {
|
||||
// Copy over fields from V16 into V17 config struct
|
||||
srvConfig := &serverConfigV17{
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "17"
|
||||
srvConfig.Credential = cv16.Credential
|
||||
@@ -1382,7 +1385,7 @@ func migrateV17ToV18() error {
|
||||
// Copy over fields from V17 into V18 config struct
|
||||
srvConfig := &serverConfigV17{
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "18"
|
||||
srvConfig.Credential = cv17.Credential
|
||||
@@ -1484,7 +1487,7 @@ func migrateV18ToV19() error {
|
||||
// Copy over fields from V18 into V19 config struct
|
||||
srvConfig := &serverConfigV18{
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "19"
|
||||
srvConfig.Credential = cv18.Credential
|
||||
@@ -1590,7 +1593,7 @@ func migrateV19ToV20() error {
|
||||
// Copy over fields from V19 into V20 config struct
|
||||
srvConfig := &serverConfigV20{
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "20"
|
||||
srvConfig.Credential = cv19.Credential
|
||||
@@ -1694,7 +1697,7 @@ func migrateV20ToV21() error {
|
||||
|
||||
// Copy over fields from V20 into V21 config struct
|
||||
srvConfig := &serverConfigV21{
|
||||
Notify: ¬ifier{},
|
||||
Notify: ¬ifierV3{},
|
||||
}
|
||||
srvConfig.Version = "21"
|
||||
srvConfig.Credential = cv20.Credential
|
||||
@@ -1798,7 +1801,7 @@ func migrateV21ToV22() error {
|
||||
|
||||
// Copy over fields from V21 into V22 config struct
|
||||
srvConfig := &serverConfigV22{
|
||||
Notify: notifier{},
|
||||
Notify: notifierV3{},
|
||||
}
|
||||
srvConfig.Version = "22"
|
||||
srvConfig.Credential = cv21.Credential
|
||||
@@ -1902,7 +1905,7 @@ func migrateV22ToV23() error {
|
||||
|
||||
// Copy over fields from V22 into V23 config struct
|
||||
srvConfig := &serverConfigV23{
|
||||
Notify: notifier{},
|
||||
Notify: notifierV3{},
|
||||
}
|
||||
srvConfig.Version = "23"
|
||||
srvConfig.Credential = cv22.Credential
|
||||
@@ -2015,7 +2018,7 @@ func migrateV23ToV24() error {
|
||||
|
||||
// Copy over fields from V23 into V24 config struct
|
||||
srvConfig := &serverConfigV24{
|
||||
Notify: notifier{},
|
||||
Notify: notifierV3{},
|
||||
}
|
||||
srvConfig.Version = "24"
|
||||
srvConfig.Credential = cv23.Credential
|
||||
@@ -2128,7 +2131,7 @@ func migrateV24ToV25() error {
|
||||
|
||||
// Copy over fields from V24 into V25 config struct
|
||||
srvConfig := &serverConfigV25{
|
||||
Notify: notifier{},
|
||||
Notify: notifierV3{},
|
||||
}
|
||||
srvConfig.Version = "25"
|
||||
srvConfig.Credential = cv24.Credential
|
||||
@@ -2246,7 +2249,7 @@ func migrateV25ToV26() error {
|
||||
|
||||
// Copy over fields from V25 into V26 config struct
|
||||
srvConfig := &serverConfigV26{
|
||||
Notify: notifier{},
|
||||
Notify: notifierV3{},
|
||||
}
|
||||
srvConfig.Version = "26"
|
||||
srvConfig.Credential = cv25.Credential
|
||||
@@ -2410,81 +2413,305 @@ func migrateV27ToV28() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Migrates '.minio.sys/config.json' to v30.
|
||||
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
|
||||
defer func() {
|
||||
// Rename config.json to config.json.deprecated only upon
|
||||
// success of this function.
|
||||
if err == nil {
|
||||
os.Rename(getConfigFile(), getConfigFile()+".deprecated")
|
||||
}
|
||||
}()
|
||||
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
// Construct path to config.json for the given bucket.
|
||||
transactionConfigFile := configFile + ".transaction"
|
||||
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
// Verify if backend already has the file.
|
||||
if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
|
||||
return err
|
||||
} // if errConfigNotFound proceed to migrate..
|
||||
|
||||
var config = &serverConfig{}
|
||||
if _, err = Load(getConfigFile(), config); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Read from deprecate file as well if necessary.
|
||||
if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return saveServerConfig(context.Background(), objAPI, config)
|
||||
}
|
||||
|
||||
// Migrates '.minio.sys/config.json' to v33.
|
||||
func migrateMinioSysConfig(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
// Construct path to config.json for the given bucket.
|
||||
transactionConfigFile := configFile + ".transaction"
|
||||
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
if err := migrateV27ToV28MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateV28ToV29MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
return migrateV29ToV30MinioSys(objAPI)
|
||||
if err := migrateV29ToV30MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateV30ToV31MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := migrateV31ToV32MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
return migrateV32ToV33MinioSys(objAPI)
|
||||
}
|
||||
|
||||
func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
|
||||
func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
|
||||
data, err := readConfig(context.Background(), objAPI, configFile)
|
||||
if err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
|
||||
var versionConfig struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
vcfg := &versionConfig
|
||||
if err = json.Unmarshal(data, vcfg); err != nil {
|
||||
return false, nil, err
|
||||
}
|
||||
return vcfg.Version == version, data, nil
|
||||
}
|
||||
|
||||
func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
srvConfig, err := readServerConfig(context.Background(), objAPI)
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "27")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if srvConfig.Version != "29" {
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
srvConfig.Version = "30"
|
||||
// Init compression config.For future migration, Compression config needs to be copied over from previous version.
|
||||
srvConfig.Compression.Enabled = false
|
||||
srvConfig.Compression.Extensions = globalCompressExtensions
|
||||
srvConfig.Compression.MimeTypes = globalCompressMimeTypes
|
||||
if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from 29 to 30 . %v", err)
|
||||
cfg := &serverConfigV28{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
|
||||
cfg.Version = "28"
|
||||
cfg.KMS = crypto.KMSConfig{}
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
srvConfig, err := readServerConfig(context.Background(), objAPI)
|
||||
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "28")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if srvConfig.Version != "28" {
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
srvConfig.Version = "29"
|
||||
if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from â28â to â29â. %v", err)
|
||||
cfg := &serverConfigV29{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Version = "29"
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
|
||||
func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
srvConfig, err := readServerConfig(context.Background(), objAPI)
|
||||
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "29")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if srvConfig.Version != "27" {
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
srvConfig.Version = "28"
|
||||
srvConfig.KMS = crypto.KMSConfig{}
|
||||
if err = saveServerConfig(context.Background(), objAPI, srvConfig); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from â27â to â28â. %v", err)
|
||||
cfg := &serverConfigV30{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
|
||||
cfg.Version = "30"
|
||||
// Init compression config.For future migration, Compression config needs to be copied over from previous version.
|
||||
cfg.Compression.Enabled = false
|
||||
cfg.Compression.Extensions = globalCompressExtensions
|
||||
cfg.Compression.MimeTypes = globalCompressMimeTypes
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "30")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := &serverConfigV31{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Version = "31"
|
||||
cfg.OpenID.JWKS = validator.JWKSArgs{
|
||||
URL: &xnet.URL{},
|
||||
}
|
||||
cfg.Policy.OPA = iampolicy.OpaArgs{
|
||||
URL: &xnet.URL{},
|
||||
AuthToken: "",
|
||||
}
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "31")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := &serverConfigV32{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Version = "32"
|
||||
cfg.Notify.NSQ = make(map[string]target.NSQArgs)
|
||||
cfg.Notify.NSQ["1"] = target.NSQArgs{}
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "32")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := &serverConfigV33{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Version = "33"
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from 32 to 33 . %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "32", "33")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
setConfigDir(rootPath)
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
@@ -77,7 +77,7 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
setConfigDir(rootPath)
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
if err := migrateV2ToV3(); err != nil {
|
||||
t.Fatal("migrate v2 to v3 should succeed when no config file is found")
|
||||
@@ -159,14 +159,15 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test if a config migration from v2 to v29 is successfully done
|
||||
func TestServerConfigMigrateV2toV29(t *testing.T) {
|
||||
// Test if a config migration from v2 to v33 is successfully done
|
||||
func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
setConfigDir(rootPath)
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
objLayer, fsDir, err := prepareFS()
|
||||
if err != nil {
|
||||
@@ -222,6 +223,7 @@ func TestServerConfigMigrateV2toV29(t *testing.T) {
|
||||
if globalServerConfig.Credential.AccessKey != accessKey {
|
||||
t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
|
||||
}
|
||||
|
||||
if globalServerConfig.Credential.SecretKey != secretKey {
|
||||
t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
|
||||
}
|
||||
@@ -234,7 +236,8 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
setConfigDir(rootPath)
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
configPath := rootPath + "/" + minioConfigFile
|
||||
|
||||
// Create a corrupted config file
|
||||
@@ -330,7 +333,8 @@ func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
setConfigDir(rootPath)
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
configPath := rootPath + "/" + minioConfigFile
|
||||
|
||||
for i := 3; i <= 17; i++ {
|
||||
|
||||
+161
-17
@@ -22,6 +22,8 @@ import (
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/event/target"
|
||||
"github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/iam/validator"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
)
|
||||
|
||||
@@ -371,7 +373,7 @@ type serverConfigV12 struct {
|
||||
Notify notifierV2 `json:"notify"`
|
||||
}
|
||||
|
||||
type notifier struct {
|
||||
type notifierV3 struct {
|
||||
AMQP map[string]target.AMQPArgs `json:"amqp"`
|
||||
Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
|
||||
Kafka map[string]target.KafkaArgs `json:"kafka"`
|
||||
@@ -396,7 +398,7 @@ type serverConfigV13 struct {
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV14 server configuration version '14' which is like
|
||||
@@ -413,7 +415,7 @@ type serverConfigV14 struct {
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV15 server configuration version '15' which is like
|
||||
@@ -430,7 +432,7 @@ type serverConfigV15 struct {
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// FileLogger is introduced to workaround the dependency about logrus
|
||||
@@ -468,7 +470,7 @@ type serverConfigV16 struct {
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV17 server configuration version '17' which is like
|
||||
@@ -487,7 +489,7 @@ type serverConfigV17 struct {
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV18 server configuration version '18' which is like
|
||||
@@ -506,7 +508,7 @@ type serverConfigV18 struct {
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV19 server configuration version '19' which is like
|
||||
@@ -524,7 +526,7 @@ type serverConfigV19 struct {
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV20 server configuration version '20' which is like
|
||||
@@ -543,7 +545,7 @@ type serverConfigV20 struct {
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV21 is just like version '20' without logger field
|
||||
@@ -558,7 +560,7 @@ type serverConfigV21 struct {
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV22 is just like version '21' with added support
|
||||
@@ -579,7 +581,7 @@ type serverConfigV22 struct {
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV23 is just like version '22' with addition of cache field.
|
||||
@@ -602,7 +604,7 @@ type serverConfigV23 struct {
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV24 is just like version '23', we had to revert
|
||||
@@ -626,7 +628,7 @@ type serverConfigV24 struct {
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV25 is just like version '24', stores additionally
|
||||
@@ -653,7 +655,7 @@ type serverConfigV25 struct {
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV26 is just like version '25', stores additionally
|
||||
@@ -677,7 +679,7 @@ type serverConfigV26 struct {
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
type loggerConsole struct {
|
||||
@@ -718,7 +720,7 @@ type serverConfigV27 struct {
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
Notify notifierV3 `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
@@ -749,12 +751,15 @@ type serverConfigV28 struct {
|
||||
KMS crypto.KMSConfig `json:"kms"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
Notify notifierV3 `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
}
|
||||
|
||||
// serverConfigV29 is just like version '28'.
|
||||
type serverConfigV29 serverConfigV28
|
||||
|
||||
// compressionConfig represents the compression settings.
|
||||
type compressionConfig struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
@@ -765,6 +770,131 @@ type compressionConfig struct {
|
||||
// serverConfigV30 is just like version '29', stores additionally
|
||||
// extensions and mimetypes fields for compression.
|
||||
type serverConfigV30 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Worm BoolFlag `json:"worm"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// KMS configuration
|
||||
KMS crypto.KMSConfig `json:"kms"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
|
||||
// Compression configuration
|
||||
Compression compressionConfig `json:"compress"`
|
||||
}
|
||||
|
||||
// serverConfigV31 is just like version '30', with OPA and OpenID configuration.
|
||||
type serverConfigV31 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Worm BoolFlag `json:"worm"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// KMS configuration
|
||||
KMS crypto.KMSConfig `json:"kms"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
|
||||
// Compression configuration
|
||||
Compression compressionConfig `json:"compress"`
|
||||
|
||||
// OpenID configuration
|
||||
OpenID struct {
|
||||
// JWKS validator config.
|
||||
JWKS validator.JWKSArgs `json:"jwks"`
|
||||
} `json:"openid"`
|
||||
|
||||
// External policy enforcements.
|
||||
Policy struct {
|
||||
// OPA configuration.
|
||||
OPA iampolicy.OpaArgs `json:"opa"`
|
||||
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
type notifier struct {
|
||||
AMQP map[string]target.AMQPArgs `json:"amqp"`
|
||||
Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
|
||||
Kafka map[string]target.KafkaArgs `json:"kafka"`
|
||||
MQTT map[string]target.MQTTArgs `json:"mqtt"`
|
||||
MySQL map[string]target.MySQLArgs `json:"mysql"`
|
||||
NATS map[string]target.NATSArgs `json:"nats"`
|
||||
NSQ map[string]target.NSQArgs `json:"nsq"`
|
||||
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
|
||||
Redis map[string]target.RedisArgs `json:"redis"`
|
||||
Webhook map[string]target.WebhookArgs `json:"webhook"`
|
||||
}
|
||||
|
||||
// serverConfigV32 is just like version '31' with added nsq notifer.
|
||||
type serverConfigV32 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Worm BoolFlag `json:"worm"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// KMS configuration
|
||||
KMS crypto.KMSConfig `json:"kms"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
|
||||
// Compression configuration
|
||||
Compression compressionConfig `json:"compress"`
|
||||
|
||||
// OpenID configuration
|
||||
OpenID struct {
|
||||
// JWKS validator config.
|
||||
JWKS validator.JWKSArgs `json:"jwks"`
|
||||
} `json:"openid"`
|
||||
|
||||
// External policy enforcements.
|
||||
Policy struct {
|
||||
// OPA configuration.
|
||||
OPA iampolicy.OpaArgs `json:"opa"`
|
||||
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS.
|
||||
type serverConfigV33 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
@@ -791,4 +921,18 @@ type serverConfigV30 struct {
|
||||
|
||||
// Compression configuration
|
||||
Compression compressionConfig `json:"compress"`
|
||||
|
||||
// OpenID configuration
|
||||
OpenID struct {
|
||||
// JWKS validator config.
|
||||
JWKS validator.JWKSArgs `json:"jwks"`
|
||||
} `json:"openid"`
|
||||
|
||||
// External policy enforcements.
|
||||
Policy struct {
|
||||
// OPA configuration.
|
||||
OPA iampolicy.OpaArgs `json:"opa"`
|
||||
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
+26
-132
@@ -20,16 +20,13 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
)
|
||||
|
||||
@@ -62,16 +59,10 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
|
||||
}
|
||||
|
||||
// Create a backup of the current config
|
||||
reader, err := readConfig(ctx, objAPI, configFile)
|
||||
oldData, err := readConfig(ctx, objAPI, configFile)
|
||||
if err == nil {
|
||||
var oldData []byte
|
||||
oldData, err = ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
backupConfigFile := path.Join(minioConfigPrefix, minioConfigBackupFile)
|
||||
err = saveConfig(objAPI, backupConfigFile, oldData)
|
||||
if err != nil {
|
||||
if err = saveConfig(ctx, objAPI, backupConfigFile, oldData); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
@@ -81,40 +72,18 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, config *serverCon
|
||||
}
|
||||
|
||||
// Save the new config in the std config path
|
||||
return saveConfig(objAPI, configFile, data)
|
||||
}
|
||||
|
||||
func readConfigEtcd(configFile string) ([]byte, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
resp, err := globalEtcdClient.Get(ctx, configFile)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Count == 0 {
|
||||
return nil, errConfigNotFound
|
||||
}
|
||||
for _, ev := range resp.Kvs {
|
||||
if string(ev.Key) == configFile {
|
||||
return ev.Value, nil
|
||||
}
|
||||
}
|
||||
return nil, errConfigNotFound
|
||||
return saveConfig(ctx, objAPI, configFile, data)
|
||||
}
|
||||
|
||||
func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, error) {
|
||||
var configData []byte
|
||||
var err error
|
||||
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
if globalEtcdClient != nil {
|
||||
configData, err = readConfigEtcd(configFile)
|
||||
configData, err = readConfigEtcd(ctx, globalEtcdClient, configFile)
|
||||
} else {
|
||||
var reader io.Reader
|
||||
reader, err = readConfig(ctx, objAPI, configFile)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
configData, err = ioutil.ReadAll(reader)
|
||||
configData, err = readConfig(ctx, objAPI, configFile)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -129,82 +98,17 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (*serverConfig, e
|
||||
}
|
||||
|
||||
var config = &serverConfig{}
|
||||
if err := json.Unmarshal(configData, config); err != nil {
|
||||
if err = json.Unmarshal(configData, config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := quick.CheckData(config); err != nil {
|
||||
if err = quick.CheckData(config); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
func checkServerConfigEtcd(configFile string) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
resp, err := globalEtcdClient.Get(ctx, configFile)
|
||||
defer cancel()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.Count == 0 {
|
||||
return errConfigNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkServerConfig(ctx context.Context, objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
if globalEtcdClient != nil {
|
||||
return checkServerConfigEtcd(configFile)
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil {
|
||||
// Convert ObjectNotFound to errConfigNotFound
|
||||
if isErrObjectNotFound(err) {
|
||||
return errConfigNotFound
|
||||
}
|
||||
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func saveConfig(objAPI ObjectLayer, configFile string, data []byte) error {
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = objAPI.PutObject(context.Background(), minioMetaBucket, configFile, hashReader, nil, ObjectOptions{})
|
||||
return err
|
||||
}
|
||||
|
||||
var errConfigNotFound = errors.New("config file not found")
|
||||
|
||||
func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) (*bytes.Buffer, error) {
|
||||
var buffer bytes.Buffer
|
||||
// Read entire content by setting size to -1
|
||||
if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
|
||||
// Convert ObjectNotFound and IncompleteBody errors into errConfigNotFound
|
||||
if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
|
||||
return nil, errConfigNotFound
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).AppendTags("configFile", configFile)
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Return config not found on empty content.
|
||||
if buffer.Len() == 0 {
|
||||
return nil, errConfigNotFound
|
||||
}
|
||||
|
||||
return &buffer, nil
|
||||
}
|
||||
|
||||
// ConfigSys - config system.
|
||||
type ConfigSys struct{}
|
||||
|
||||
@@ -234,7 +138,8 @@ func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
|
||||
case _ = <-retryTimerCh:
|
||||
err := initConfig(objAPI)
|
||||
if err != nil {
|
||||
if isInsufficientReadQuorum(err) || isInsufficientWriteQuorum(err) {
|
||||
if strings.Contains(err.Error(), InsufficientReadQuorum{}.Error()) ||
|
||||
strings.Contains(err.Error(), InsufficientWriteQuorum{}.Error()) {
|
||||
logger.Info("Waiting for configuration to be initialized..")
|
||||
continue
|
||||
}
|
||||
@@ -251,29 +156,6 @@ func NewConfigSys() *ConfigSys {
|
||||
return &ConfigSys{}
|
||||
}
|
||||
|
||||
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
func migrateConfigToMinioSys(objAPI ObjectLayer) error {
|
||||
defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
|
||||
|
||||
// Verify if backend already has the file.
|
||||
if err := checkServerConfig(context.Background(), objAPI); err != errConfigNotFound {
|
||||
return err
|
||||
} // if errConfigNotFound proceed to migrate..
|
||||
|
||||
var config = &serverConfig{}
|
||||
if _, err := Load(getConfigFile(), config); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Read from deprecate file as well if necessary.
|
||||
if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return saveServerConfig(context.Background(), objAPI, config)
|
||||
}
|
||||
|
||||
// Initialize and load config from remote etcd or local config directory
|
||||
func initConfig(objAPI ObjectLayer) error {
|
||||
if objAPI == nil {
|
||||
@@ -285,7 +167,14 @@ func initConfig(objAPI ObjectLayer) error {
|
||||
resp, err := globalEtcdClient.Get(ctx, getConfigFile())
|
||||
cancel()
|
||||
if err == nil && resp.Count > 0 {
|
||||
return migrateConfig()
|
||||
if err = migrateConfig(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Migrates etcd ${HOME}/.minio/config.json to '/config/config.json'
|
||||
if err := migrateConfigToMinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if isFile(getConfigFile()) {
|
||||
@@ -301,10 +190,15 @@ func initConfig(objAPI ObjectLayer) error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := checkServerConfig(context.Background(), objAPI); err != nil {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
// Watch config for changes and reloads them in-memory.
|
||||
go watchConfig(objAPI, configFile, loadConfig)
|
||||
|
||||
if err := checkConfig(context.Background(), objAPI, configFile); err != nil {
|
||||
if err == errConfigNotFound {
|
||||
// Config file does not exist, we create it fresh and return upon success.
|
||||
if err = newConfig(objAPI); err != nil {
|
||||
if err = newSrvConfig(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
|
||||
@@ -22,16 +22,16 @@ import (
|
||||
)
|
||||
|
||||
// Writes S3 compatible copy part range error.
|
||||
func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL) {
|
||||
func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL, browser bool) {
|
||||
switch err {
|
||||
case errInvalidRange:
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRange, url)
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRange, url, browser)
|
||||
return
|
||||
case errInvalidRangeSource:
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url)
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url, browser)
|
||||
return
|
||||
default:
|
||||
writeErrorResponse(w, ErrInternalError, url)
|
||||
writeErrorResponse(w, ErrInternalError, url, browser)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,5 +16,6 @@ package crypto
|
||||
|
||||
// KMSConfig has the KMS config for hashicorp vault
|
||||
type KMSConfig struct {
|
||||
Vault VaultConfig `json:"vault"`
|
||||
AutoEncryption bool `json:"-"`
|
||||
Vault VaultConfig `json:"vault"`
|
||||
}
|
||||
|
||||
@@ -95,7 +95,7 @@ var ssecIsRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
}{
|
||||
{Header: http.Header{}, Expected: false}, // 0
|
||||
{Header: http.Header{}, Expected: false}, // 0
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true}, // 1
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true}, // 3
|
||||
@@ -137,7 +137,7 @@ var ssecCopyIsRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
}{
|
||||
{Header: http.Header{}, Expected: false}, // 0
|
||||
{Header: http.Header{}, Expected: false}, // 0
|
||||
{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true}, // 1
|
||||
{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2
|
||||
{Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true}, // 3
|
||||
|
||||
@@ -140,3 +140,37 @@ func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) {
|
||||
mac.Sum(partKey[:0])
|
||||
return partKey
|
||||
}
|
||||
|
||||
// SealETag seals the etag using the object key.
|
||||
// It does not encrypt empty ETags because such ETags indicate
|
||||
// that the S3 client hasn't sent an ETag = MD5(object) and
|
||||
// the backend can pick an ETag value.
|
||||
func (key ObjectKey) SealETag(etag []byte) []byte {
|
||||
if len(etag) == 0 { // don't encrypt empty ETag - only if client sent ETag = MD5(object)
|
||||
return etag
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
mac := hmac.New(sha256.New, key[:])
|
||||
mac.Write([]byte("SSE-etag"))
|
||||
if _, err := sio.Encrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
|
||||
logger.CriticalIf(context.Background(), errors.New("Unable to encrypt ETag using object key"))
|
||||
}
|
||||
return buffer.Bytes()
|
||||
}
|
||||
|
||||
// UnsealETag unseals the etag using the provided object key.
|
||||
// It does not try to decrypt the ETag if len(etag) == 16
|
||||
// because such ETags indicate that the S3 client hasn't sent
|
||||
// an ETag = MD5(object) and the backend has picked an ETag value.
|
||||
func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
|
||||
if !IsETagSealed(etag) {
|
||||
return etag, nil
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
mac := hmac.New(sha256.New, key[:])
|
||||
mac.Write([]byte("SSE-etag"))
|
||||
if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buffer.Bytes(), nil
|
||||
}
|
||||
|
||||
@@ -166,3 +166,31 @@ func TestDerivePartKey(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sealUnsealETagTests = []string{
|
||||
"",
|
||||
"90682b8e8cc7609c",
|
||||
"90682b8e8cc7609c4671e1d64c73fc30",
|
||||
"90682b8e8cc7609c4671e1d64c73fc307fb3104f",
|
||||
}
|
||||
|
||||
func TestSealETag(t *testing.T) {
|
||||
var key ObjectKey
|
||||
for i := range key {
|
||||
key[i] = byte(i)
|
||||
}
|
||||
for i, etag := range sealUnsealETagTests {
|
||||
tag, err := hex.DecodeString(etag)
|
||||
if err != nil {
|
||||
t.Errorf("Test %d: failed to decode etag: %s", i, err)
|
||||
}
|
||||
sealedETag := key.SealETag(tag)
|
||||
unsealedETag, err := key.UnsealETag(sealedETag)
|
||||
if err != nil {
|
||||
t.Errorf("Test %d: failed to decrypt etag: %s", i, err)
|
||||
}
|
||||
if !bytes.Equal(unsealedETag, tag) {
|
||||
t.Errorf("Test %d: unsealed etag does not match: got %s - want %s", i, hex.EncodeToString(unsealedETag), etag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,6 +40,26 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
|
||||
delete(metadata, SSECopyKey)
|
||||
}
|
||||
|
||||
// RemoveSSEHeaders removes all crypto-specific SSE
|
||||
// header entries from the metadata map.
|
||||
func RemoveSSEHeaders(metadata map[string]string) {
|
||||
delete(metadata, SSEHeader)
|
||||
delete(metadata, SSECKeyMD5)
|
||||
delete(metadata, SSECAlgorithm)
|
||||
}
|
||||
|
||||
// RemoveInternalEntries removes all crypto-specific internal
|
||||
// metadata entries from the metadata map.
|
||||
func RemoveInternalEntries(metadata map[string]string) {
|
||||
delete(metadata, SSEMultipart)
|
||||
delete(metadata, SSEIV)
|
||||
delete(metadata, SSESealAlgorithm)
|
||||
delete(metadata, SSECSealedKey)
|
||||
delete(metadata, S3SealedKey)
|
||||
delete(metadata, S3KMSKeyID)
|
||||
delete(metadata, S3KMSSealedKey)
|
||||
}
|
||||
|
||||
// IsEncrypted returns true if the object metadata indicates
|
||||
// that it was uploaded using some form of server-side-encryption.
|
||||
//
|
||||
@@ -219,3 +239,6 @@ func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err
|
||||
copy(sealedKey.Key[:], encryptedKey)
|
||||
return sealedKey, nil
|
||||
}
|
||||
|
||||
// IsETagSealed returns true if the etag seems to be encrypted.
|
||||
func IsETagSealed(etag []byte) bool { return len(etag) > 16 }
|
||||
|
||||
@@ -17,6 +17,7 @@ package crypto
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -364,3 +365,75 @@ func TestSSECCreateMetadata(t *testing.T) {
|
||||
}()
|
||||
_ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm})
|
||||
}
|
||||
|
||||
var isETagSealedTests = []struct {
|
||||
ETag string
|
||||
IsSealed bool
|
||||
}{
|
||||
{ETag: "", IsSealed: false}, // 0
|
||||
{ETag: "90682b8e8cc7609c4671e1d64c73fc30", IsSealed: false}, // 1
|
||||
{ETag: "f201040c9dc593e39ea004dc1323699bcd", IsSealed: true}, // 2 not valid ciphertext but looks like sealed ETag
|
||||
{ETag: "20000f00fba2ee2ae4845f725964eeb9e092edfabc7ab9f9239e8344341f769a51ce99b4801b0699b92b16a72fa94972", IsSealed: true}, // 3
|
||||
}
|
||||
|
||||
func TestIsETagSealed(t *testing.T) {
|
||||
for i, test := range isETagSealedTests {
|
||||
etag, err := hex.DecodeString(test.ETag)
|
||||
if err != nil {
|
||||
t.Errorf("Test %d: failed to decode etag: %s", i, err)
|
||||
}
|
||||
if sealed := IsETagSealed(etag); sealed != test.IsSealed {
|
||||
t.Errorf("Test %d: got %v - want %v", i, sealed, test.IsSealed)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var removeInternalEntriesTests = []struct {
|
||||
Metadata, Expected map[string]string
|
||||
}{
|
||||
{ // 0
|
||||
Metadata: map[string]string{
|
||||
SSEMultipart: "",
|
||||
SSEIV: "",
|
||||
SSESealAlgorithm: "",
|
||||
SSECSealedKey: "",
|
||||
S3SealedKey: "",
|
||||
S3KMSKeyID: "",
|
||||
S3KMSSealedKey: "",
|
||||
},
|
||||
Expected: map[string]string{},
|
||||
},
|
||||
{ // 1
|
||||
Metadata: map[string]string{
|
||||
SSEMultipart: "",
|
||||
SSEIV: "",
|
||||
"X-Amz-Meta-A": "X",
|
||||
"X-Minio-Internal-B": "Y",
|
||||
},
|
||||
Expected: map[string]string{
|
||||
"X-Amz-Meta-A": "X",
|
||||
"X-Minio-Internal-B": "Y",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestRemoveInternalEntries(t *testing.T) {
|
||||
isEqual := func(x, y map[string]string) bool {
|
||||
if len(x) != len(y) {
|
||||
return false
|
||||
}
|
||||
for k, v := range x {
|
||||
if u, ok := y[k]; !ok || v != u {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
for i, test := range removeInternalEntriesTests {
|
||||
RemoveInternalEntries(test.Metadata)
|
||||
if !isEqual(test.Metadata, test.Expected) {
|
||||
t.Errorf("Test %d: got %v - want %v", i, test.Metadata, test.Expected)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
@@ -69,10 +71,59 @@ const (
|
||||
// domain is "SSE-S3".
|
||||
func (s3) String() string { return "SSE-S3" }
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using KMS and returns the decrypted object
|
||||
// key.
|
||||
func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
|
||||
return
|
||||
}
|
||||
|
||||
// String returns the SSE domain as string. For SSE-C the
|
||||
// domain is "SSE-C".
|
||||
func (ssec) String() string { return "SSE-C" }
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using the SSE-C client key of the HTTP headers
|
||||
// and returns the decrypted object key.
|
||||
func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
clientKey, err := sse.ParseHTTP(h)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return unsealObjectKey(clientKey, metadata, bucket, object)
|
||||
}
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using the SSE-Copy client key of the HTTP headers
|
||||
// and returns the decrypted object key.
|
||||
func (sse ssecCopy) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
clientKey, err := sse.ParseHTTP(h)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return unsealObjectKey(clientKey, metadata, bucket, object)
|
||||
}
|
||||
|
||||
// unsealObjectKey decrypts and returns the sealed object key
|
||||
// from the metadata using the SSE-C client key.
|
||||
func unsealObjectKey(clientKey [32]byte, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
sealedKey, err := SSEC.ParseMetadata(metadata)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = key.Unseal(clientKey, sealedKey, SSEC.String(), bucket, object)
|
||||
return
|
||||
}
|
||||
|
||||
// EncryptSinglePart encrypts an io.Reader which must be the
|
||||
// the body of a single-part PUT request.
|
||||
func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader {
|
||||
|
||||
+196
-1
@@ -14,7 +14,10 @@
|
||||
|
||||
package crypto
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestS3String(t *testing.T) {
|
||||
const Domain = "SSE-S3"
|
||||
@@ -29,3 +32,195 @@ func TestSSECString(t *testing.T) {
|
||||
t.Errorf("SSEC's string method returns wrong domain: got '%s' - want '%s'", domain, Domain)
|
||||
}
|
||||
}
|
||||
|
||||
var ssecUnsealObjectKeyTests = []struct {
|
||||
Headers http.Header
|
||||
Bucket, Object string
|
||||
Metadata map[string]string
|
||||
|
||||
ExpectedErr error
|
||||
}{
|
||||
{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
|
||||
Headers: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: nil,
|
||||
},
|
||||
{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
|
||||
Headers: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object2",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: ErrSecretKeyMismatch,
|
||||
},
|
||||
{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
|
||||
Headers: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: errMissingInternalSealAlgorithm,
|
||||
},
|
||||
{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
|
||||
Headers: http.Header{
|
||||
"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: ErrMissingCustomerKeyMD5,
|
||||
},
|
||||
}
|
||||
|
||||
func TestSSECUnsealObjectKey(t *testing.T) {
|
||||
for i, test := range ssecUnsealObjectKeyTests {
|
||||
if _, err := SSEC.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
|
||||
t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sseCopyUnsealObjectKeyTests = []struct {
|
||||
Headers http.Header
|
||||
Bucket, Object string
|
||||
Metadata map[string]string
|
||||
|
||||
ExpectedErr error
|
||||
}{
|
||||
{ // 0 - Valid HTTP headers and valid metadata entries for bucket/object
|
||||
Headers: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: nil,
|
||||
},
|
||||
{ // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2
|
||||
Headers: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object2",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: ErrSecretKeyMismatch,
|
||||
},
|
||||
{ // 2 - Valid HTTP headers but invalid metadata entries for bucket/object
|
||||
Headers: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: errMissingInternalSealAlgorithm,
|
||||
},
|
||||
{ // 3 - Invalid HTTP headers for valid metadata entries for bucket/object
|
||||
Headers: http.Header{
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"},
|
||||
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
},
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=",
|
||||
},
|
||||
ExpectedErr: ErrMissingCustomerKeyMD5,
|
||||
},
|
||||
}
|
||||
|
||||
func TestSSECopyUnsealObjectKey(t *testing.T) {
|
||||
for i, test := range sseCopyUnsealObjectKeyTests {
|
||||
if _, err := SSECopy.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
|
||||
t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var s3UnsealObjectKeyTests = []struct {
|
||||
KMS KMS
|
||||
Bucket, Object string
|
||||
Metadata map[string]string
|
||||
|
||||
ExpectedErr error
|
||||
}{
|
||||
{ // 0 - Valid KMS key-ID and valid metadata entries for bucket/object
|
||||
KMS: NewKMS([32]byte{}),
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
|
||||
"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "test-key-1",
|
||||
"X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256",
|
||||
},
|
||||
ExpectedErr: nil,
|
||||
},
|
||||
{ // 1 - Valid KMS key-ID for invalid metadata entries for bucket/object
|
||||
KMS: NewKMS([32]byte{}),
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
Metadata: map[string]string{
|
||||
"X-Minio-Internal-Server-Side-Encryption-Iv": "hhVY0LKR1YtZbzAKxTWUfZt5enDfYX6Fxz1ma8Kiudc=",
|
||||
"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfALhsOeD5AE3s5Zgq3DZ5VFGsOa3B0ksVC86veDcaj+fXv2U0VadhPaOKYr9Emd5ssOsO0uIhIIrKiOy9rA==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAMRS2iw45FsfiF3QXajSYVWj1lxMpQm6DxDGPtADCX6fJQQ4atHBtfpgqJFyeQmIHsm0FBI+UlHw1Lv4ug==",
|
||||
"X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "test-key-1",
|
||||
},
|
||||
ExpectedErr: errMissingInternalSealAlgorithm,
|
||||
},
|
||||
}
|
||||
|
||||
func TestS3UnsealObjectKey(t *testing.T) {
|
||||
for i, test := range s3UnsealObjectKeyTests {
|
||||
if _, err := S3.UnsealObjectKey(test.KMS, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr {
|
||||
t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+123
-179
@@ -19,195 +19,133 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
vault "github.com/hashicorp/vault/api"
|
||||
)
|
||||
|
||||
const (
|
||||
// vaultEndpointEnv Vault endpoint environment variable
|
||||
vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
|
||||
// vaultAuthTypeEnv type of vault auth to be used
|
||||
vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
|
||||
// vaultAppRoleIDEnv Vault AppRole ID environment variable
|
||||
vaultAppRoleIDEnv = "MINIO_SSE_VAULT_APPROLE_ID"
|
||||
// vaultAppSecretIDEnv Vault AppRole Secret environment variable
|
||||
vaultAppSecretIDEnv = "MINIO_SSE_VAULT_APPROLE_SECRET"
|
||||
// vaultKeyVersionEnv Vault Key Version environment variable
|
||||
vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
|
||||
// vaultKeyNameEnv Vault Encryption Key Name environment variable
|
||||
vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
|
||||
|
||||
// vaultCAPath is the path to a directory of PEM-encoded CA
|
||||
// cert files to verify the Vault server SSL certificate.
|
||||
vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
|
||||
)
|
||||
|
||||
var (
|
||||
//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
|
||||
ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
|
||||
)
|
||||
|
||||
// VaultKey represents vault encryption key-ring.
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"` // The name of the encryption key-ring
|
||||
Version int `json:"version"` // The key version
|
||||
}
|
||||
|
||||
// VaultAuth represents vault authentication type.
|
||||
// Currently the only supported authentication type is AppRole.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"` // The authentication type
|
||||
AppRole VaultAppRole `json:"approle"` // The AppRole authentication credentials
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault AppRole authentication credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"` // The AppRole access ID
|
||||
Secret string `json:"secret"` // The AppRole secret
|
||||
}
|
||||
|
||||
// VaultConfig represents vault configuration.
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"` // The vault API endpoint as URL
|
||||
CAPath string `json:"-"` // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
|
||||
Auth VaultAuth `json:"auth"` // The vault authentication configuration
|
||||
Key VaultKey `json:"key-id"` // The named key used for key-generation / decryption.
|
||||
Namespace string `json:"-"` // The vault namespace of enterprise vault instances
|
||||
}
|
||||
|
||||
// vaultService represents a connection to a vault KMS.
|
||||
type vaultService struct {
|
||||
config *VaultConfig
|
||||
client *vault.Client
|
||||
leaseDuration time.Duration
|
||||
}
|
||||
|
||||
// return transit secret engine's path for generate data key operation
|
||||
func (v *vaultService) genDataKeyEndpoint(key string) string {
|
||||
return "/transit/datakey/plaintext/" + key
|
||||
var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
|
||||
|
||||
// empty/default vault configuration used to check whether a particular is empty.
|
||||
var emptyVaultConfig = VaultConfig{}
|
||||
|
||||
// IsEmpty returns true if the vault config struct is an
|
||||
// empty configuration.
|
||||
func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }
|
||||
|
||||
// Verify returns a nil error if the vault configuration
|
||||
// is valid. A valid configuration is either empty or
|
||||
// contains valid non-default values.
|
||||
func (v *VaultConfig) Verify() (err error) {
|
||||
if v.IsEmpty() {
|
||||
return // an empty configuration is valid
|
||||
}
|
||||
switch {
|
||||
case v.Endpoint == "":
|
||||
err = errors.New("crypto: missing hashicorp vault endpoint")
|
||||
case strings.ToLower(v.Auth.Type) != "approle":
|
||||
err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
|
||||
case v.Auth.AppRole.ID == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppRole ID")
|
||||
case v.Auth.AppRole.Secret == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppSecret ID")
|
||||
case v.Key.Name == "":
|
||||
err = errors.New("crypto: missing hashicorp vault key name")
|
||||
case v.Key.Version < 0:
|
||||
err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// return transit secret engine's path for decrypt operation
|
||||
func (v *vaultService) decryptEndpoint(key string) string {
|
||||
return "/transit/decrypt/" + key
|
||||
}
|
||||
// NewVault initializes Hashicorp Vault KMS by authenticating
|
||||
// to Vault with the credentials in config and gets a client
|
||||
// token for future api calls.
|
||||
func NewVault(config VaultConfig) (KMS, error) {
|
||||
if config.IsEmpty() {
|
||||
return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
|
||||
}
|
||||
if err := config.Verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// VaultKey represents vault encryption key-id name & version
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
}
|
||||
vaultCfg := vault.Config{Address: config.Endpoint}
|
||||
if err := vaultCfg.ConfigureTLS(&vault.TLSConfig{CAPath: config.CAPath}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := vault.NewClient(&vaultCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.Namespace != "" {
|
||||
client.SetNamespace(config.Namespace)
|
||||
}
|
||||
|
||||
// VaultAuth represents vault auth type to use. For now, AppRole is the only supported
|
||||
// auth type.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"`
|
||||
AppRole VaultAppRole `json:"approle"`
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault approle credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"`
|
||||
Secret string `json:"secret"`
|
||||
}
|
||||
|
||||
// VaultConfig holds config required to start vault service
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Auth VaultAuth `json:"auth"`
|
||||
Key VaultKey `json:"key-id"`
|
||||
}
|
||||
|
||||
// validate whether all required env variables needed to start vault service have
|
||||
// been set
|
||||
func validateVaultConfig(c *VaultConfig) error {
|
||||
if c.Endpoint == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
|
||||
payload := map[string]interface{}{
|
||||
"role_id": config.Auth.AppRole.ID,
|
||||
"secret_id": config.Auth.AppRole.Secret,
|
||||
}
|
||||
if strings.ToLower(c.Auth.Type) != "approle" {
|
||||
return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
|
||||
}
|
||||
if c.Auth.AppRole.ID == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppRole ID - %s is empty", vaultAppRoleIDEnv)
|
||||
}
|
||||
if c.Auth.AppRole.Secret == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppSecret ID - %s is empty", vaultAppSecretIDEnv)
|
||||
}
|
||||
if c.Key.Name == "" {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyNameEnv)
|
||||
}
|
||||
if c.Key.Version < 0 {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// authenticate to vault with app role id and app role secret, and get a client access token, lease duration
|
||||
func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (token string, duration int, err error) {
|
||||
data := map[string]interface{}{
|
||||
"role_id": appRoleID,
|
||||
"secret_id": appSecret,
|
||||
}
|
||||
resp, e := client.Logical().Write("auth/approle/login", data)
|
||||
if e != nil {
|
||||
return token, duration, e
|
||||
resp, err := client.Logical().Write("auth/approle/login", payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Auth == nil {
|
||||
return token, duration, ErrKMSAuthLogin
|
||||
return nil, ErrKMSAuthLogin
|
||||
}
|
||||
return resp.Auth.ClientToken, resp.Auth.LeaseDuration, nil
|
||||
|
||||
client.SetToken(resp.Auth.ClientToken)
|
||||
v := &vaultService{client: client, config: &config, leaseDuration: time.Duration(resp.Auth.LeaseDuration)}
|
||||
v.renewToken()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// NewVaultConfig sets KMSConfig from environment
|
||||
// variables and performs validations.
|
||||
func NewVaultConfig() (KMSConfig, error) {
|
||||
kc := KMSConfig{}
|
||||
endpoint := os.Getenv(vaultEndpointEnv)
|
||||
roleID := os.Getenv(vaultAppRoleIDEnv)
|
||||
roleSecret := os.Getenv(vaultAppSecretIDEnv)
|
||||
keyName := os.Getenv(vaultKeyNameEnv)
|
||||
keyVersion := 0
|
||||
authType := "approle"
|
||||
if versionStr := os.Getenv(vaultKeyVersionEnv); versionStr != "" {
|
||||
version, err := strconv.Atoi(versionStr)
|
||||
if err != nil {
|
||||
return kc, fmt.Errorf("Unable to parse %s value (`%s`)", vaultKeyVersionEnv, versionStr)
|
||||
}
|
||||
keyVersion = version
|
||||
}
|
||||
// return if none of the vault env variables are configured
|
||||
if (endpoint == "") && (roleID == "") && (roleSecret == "") && (keyName == "") && (keyVersion == 0) {
|
||||
return kc, nil
|
||||
}
|
||||
c := VaultConfig{
|
||||
Endpoint: endpoint,
|
||||
Auth: VaultAuth{
|
||||
Type: authType,
|
||||
AppRole: VaultAppRole{
|
||||
ID: roleID,
|
||||
Secret: roleSecret,
|
||||
},
|
||||
},
|
||||
Key: VaultKey{
|
||||
Version: keyVersion,
|
||||
Name: keyName,
|
||||
},
|
||||
}
|
||||
if err := validateVaultConfig(&c); err != nil {
|
||||
return kc, err
|
||||
}
|
||||
kc.Vault = c
|
||||
return kc, nil
|
||||
}
|
||||
|
||||
// NewVault initializes Hashicorp Vault KMS by
|
||||
// authenticating to Vault with the credentials in KMSConfig,
|
||||
// and gets a client token for future api calls.
|
||||
func NewVault(kmsConf KMSConfig) (KMS, error) {
|
||||
config := kmsConf.Vault
|
||||
vconfig := &vault.Config{
|
||||
Address: config.Endpoint,
|
||||
}
|
||||
if err := vconfig.ConfigureTLS(&vault.TLSConfig{
|
||||
CAPath: os.Getenv(vaultCAPath),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := vault.NewClient(vconfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accessToken, leaseDuration, err := getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// authenticate and get the access token
|
||||
c.SetToken(accessToken)
|
||||
v := vaultService{client: c, config: &config, leaseDuration: time.Duration(leaseDuration)}
|
||||
v.renewToken(c)
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (v *vaultService) renewToken(c *vault.Client) {
|
||||
// renewToken starts a new go-routine which renews
|
||||
// the vault authentication token periodically.
|
||||
func (v *vaultService) renewToken() {
|
||||
retryDelay := 1 * time.Minute
|
||||
go func() {
|
||||
for {
|
||||
s, err := c.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
if err != nil {
|
||||
time.Sleep(retryDelay)
|
||||
continue
|
||||
@@ -218,48 +156,54 @@ func (v *vaultService) renewToken(c *vault.Client) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Generates a random plain text key, sealed plain text key from
|
||||
// Vault. It returns the plaintext key and sealed plaintext key on success
|
||||
// GenerateKey returns a new plaintext key, generated by the KMS,
|
||||
// and a sealed version of this plaintext key encrypted using the
|
||||
// named key referenced by keyID. It also binds the generated key
|
||||
// cryptographically to the provided context.
|
||||
func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.genDataKeyEndpoint(keyID), payload)
|
||||
|
||||
if err1 != nil {
|
||||
return key, sealedKey, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, sealedKey, err
|
||||
}
|
||||
sealKey := s.Data["ciphertext"].(string)
|
||||
plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
|
||||
if err != nil {
|
||||
return key, sealedKey, err1
|
||||
return key, sealedKey, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
return key, []byte(sealKey), nil
|
||||
}
|
||||
|
||||
// unsealKMSKey unseals the sealedKey using the Vault master key
|
||||
// referenced by the keyID. The plain text key is returned on success.
|
||||
// UnsealKey returns the decrypted sealedKey as plaintext key.
|
||||
// Therefore it sends the sealedKey to the KMS which decrypts
|
||||
// it using the named key referenced by keyID and responses with
|
||||
// the plaintext key.
|
||||
//
|
||||
// The context must be same context as the one provided while
|
||||
// generating the plaintext key / sealedKey.
|
||||
func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"ciphertext": string(sealedKey),
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.decryptEndpoint(keyID), payload)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
base64Key := s.Data["plaintext"].(string)
|
||||
plainKey, err1 := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
plainKey, err := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
@@ -30,7 +30,6 @@ import (
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
@@ -92,7 +91,7 @@ func newCacheFSObjects(dir string, expiry int, maxDiskUsagePct int) (*cacheFSObj
|
||||
appendFileMap: make(map[string]*fsAppendFile),
|
||||
}
|
||||
|
||||
go fsObjects.cleanupStaleMultipartUploads(context.Background(), globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
|
||||
go fsObjects.cleanupStaleMultipartUploads(context.Background(), GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
|
||||
|
||||
cacheFS := cacheFSObjects{
|
||||
FSObjects: fsObjects,
|
||||
@@ -159,7 +158,7 @@ func (cfs *cacheFSObjects) purgeTrash() {
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-globalServiceDoneCh:
|
||||
case <-GlobalServiceDoneCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
trashPath := path.Join(cfs.fsPath, minioMetaBucket, cacheTrashDir)
|
||||
@@ -258,7 +257,7 @@ func (cfs *cacheFSObjects) IsOnline() bool {
|
||||
}
|
||||
|
||||
// Caches the object to disk
|
||||
func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) error {
|
||||
func (cfs *cacheFSObjects) Put(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) error {
|
||||
if cfs.diskUsageHigh() {
|
||||
select {
|
||||
case cfs.purgeChan <- struct{}{}:
|
||||
@@ -301,7 +300,8 @@ func (cfs *cacheFSObjects) Exists(ctx context.Context, bucket, object string) bo
|
||||
|
||||
// Identical to fs PutObject operation except that it uses ETag in metadata
|
||||
// headers.
|
||||
func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
|
||||
func (cfs *cacheFSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
|
||||
data := r.Reader
|
||||
fs := cfs.FSObjects
|
||||
// Lock the object.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
|
||||
+57
-55
@@ -38,8 +38,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// disk cache needs to have cacheSizeMultiplier * object size space free for a cache entry to be created.
|
||||
cacheSizeMultiplier = 100
|
||||
// disk cache needs to have object size space free for a cache entry to be created.
|
||||
cacheTrashDir = "trash"
|
||||
cacheCleanupInterval = 10 // in minutes
|
||||
)
|
||||
@@ -61,16 +60,16 @@ type cacheObjects struct {
|
||||
GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
|
||||
GetObjectFn func(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
|
||||
GetObjectInfoFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
PutObjectFn func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
PutObjectFn func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
DeleteObjectFn func(ctx context.Context, bucket, object string) error
|
||||
ListObjectsFn func(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
|
||||
ListObjectsV2Fn func(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
|
||||
ListBucketsFn func(ctx context.Context) (buckets []BucketInfo, err error)
|
||||
GetBucketInfoFn func(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
|
||||
NewMultipartUploadFn func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
|
||||
PutObjectPartFn func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
|
||||
PutObjectPartFn func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
|
||||
AbortMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string) error
|
||||
CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
|
||||
CompleteMultipartUploadFn func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
DeleteBucketFn func(ctx context.Context, bucket string) error
|
||||
}
|
||||
|
||||
@@ -93,14 +92,14 @@ type CacheObjectLayer interface {
|
||||
GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
|
||||
GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
|
||||
GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
DeleteObject(ctx context.Context, bucket, object string) error
|
||||
|
||||
// Multipart operations.
|
||||
NewMultipartUpload(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error)
|
||||
PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error)
|
||||
PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error)
|
||||
AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string) error
|
||||
CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error)
|
||||
CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
|
||||
// Storage operations.
|
||||
StorageInfo(ctx context.Context) CacheStorageInfo
|
||||
@@ -184,10 +183,7 @@ func (c cacheObjects) getMetadata(objInfo ObjectInfo) map[string]string {
|
||||
}
|
||||
|
||||
func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
||||
|
||||
objInfo, bkErr := c.GetObjectInfoFn(ctx, bucket, object, opts)
|
||||
|
||||
if c.isCacheExclude(bucket, object) || !objInfo.IsCacheable() {
|
||||
if c.isCacheExclude(bucket, object) {
|
||||
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
|
||||
}
|
||||
|
||||
@@ -197,30 +193,29 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
|
||||
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
|
||||
}
|
||||
|
||||
backendDown := backendDownError(bkErr)
|
||||
if bkErr != nil && !backendDown {
|
||||
cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
||||
|
||||
objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
|
||||
if backendDownError(err) && cacheErr == nil {
|
||||
return cacheReader, nil
|
||||
} else if err != nil {
|
||||
if _, ok := err.(ObjectNotFound); ok {
|
||||
// Delete the cached entry if backend object was deleted.
|
||||
// Delete cached entry if backend object was deleted.
|
||||
dcache.Delete(ctx, bucket, object)
|
||||
}
|
||||
return nil, bkErr
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if !backendDown && filterFromCache(objInfo.UserDefined) {
|
||||
if !objInfo.IsCacheable() || filterFromCache(objInfo.UserDefined) {
|
||||
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
|
||||
}
|
||||
|
||||
if cacheReader, cacheErr := dcache.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts); cacheErr == nil {
|
||||
if backendDown {
|
||||
// If the backend is down, serve the request from cache.
|
||||
return cacheReader, nil
|
||||
}
|
||||
|
||||
if cacheErr == nil {
|
||||
if cacheReader.ObjInfo.ETag == objInfo.ETag && !isStaleCache(objInfo) {
|
||||
// Object is not stale, so serve from cache
|
||||
return cacheReader, nil
|
||||
}
|
||||
|
||||
cacheReader.Close()
|
||||
// Object is stale, so delete from cache
|
||||
dcache.Delete(ctx, bucket, object)
|
||||
}
|
||||
@@ -232,8 +227,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
|
||||
// We don't cache partial objects.
|
||||
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
|
||||
}
|
||||
if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
|
||||
// cache only objects < 1/100th of disk capacity
|
||||
if !dcache.diskAvailable(objInfo.Size) {
|
||||
return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, writeLock, opts)
|
||||
}
|
||||
|
||||
@@ -253,7 +247,7 @@ func (c cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
|
||||
|
||||
go func() {
|
||||
opts := ObjectOptions{}
|
||||
putErr := dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(bkReader.ObjInfo), opts)
|
||||
putErr := dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(bkReader.ObjInfo), opts)
|
||||
// close the write end of the pipe, so the error gets
|
||||
// propagated to getObjReader
|
||||
pipeWriter.CloseWithError(putErr)
|
||||
@@ -290,6 +284,10 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
|
||||
return err
|
||||
}
|
||||
|
||||
if !backendDown && !objInfo.IsCacheable() {
|
||||
return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
|
||||
}
|
||||
|
||||
if !backendDown && filterFromCache(objInfo.UserDefined) {
|
||||
return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
|
||||
}
|
||||
@@ -305,12 +303,11 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
|
||||
}
|
||||
dcache.Delete(ctx, bucket, object)
|
||||
}
|
||||
if startOffset != 0 || length != objInfo.Size {
|
||||
if startOffset != 0 || (length > 0 && length != objInfo.Size) {
|
||||
// We don't cache partial objects.
|
||||
return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
|
||||
}
|
||||
if !dcache.diskAvailable(objInfo.Size * cacheSizeMultiplier) {
|
||||
// cache only objects < 1/100th of disk capacity
|
||||
if !dcache.diskAvailable(objInfo.Size) {
|
||||
return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
|
||||
}
|
||||
// Initialize pipe.
|
||||
@@ -323,7 +320,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
|
||||
gerr := GetObjectFn(ctx, bucket, object, 0, objInfo.Size, io.MultiWriter(writer, pipeWriter), etag, opts)
|
||||
pipeWriter.CloseWithError(gerr) // Close writer explicitly signaling we wrote all data.
|
||||
}()
|
||||
err = dcache.Put(ctx, bucket, object, hashReader, c.getMetadata(objInfo), opts)
|
||||
err = dcache.Put(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), c.getMetadata(objInfo), opts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -647,8 +644,9 @@ func (c cacheObjects) isCacheExclude(bucket, object string) bool {
|
||||
}
|
||||
|
||||
// PutObject - caches the uploaded object for single Put operations
|
||||
func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
putObjectFn := c.PutObjectFn
|
||||
data := r.Reader
|
||||
dcache, err := c.cache.getCacheFS(ctx, bucket, object)
|
||||
if err != nil {
|
||||
// disk cache could not be located,execute backend call.
|
||||
@@ -657,7 +655,7 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
|
||||
size := r.Size()
|
||||
|
||||
// fetch from backend if there is no space on cache drive
|
||||
if !dcache.diskAvailable(size * cacheSizeMultiplier) {
|
||||
if !dcache.diskAvailable(size) {
|
||||
return putObjectFn(ctx, bucket, object, r, metadata, opts)
|
||||
}
|
||||
// fetch from backend if cache exclude pattern or cache-control
|
||||
@@ -669,20 +667,20 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
|
||||
objInfo = ObjectInfo{}
|
||||
// Initialize pipe to stream data to backend
|
||||
pipeReader, pipeWriter := io.Pipe()
|
||||
hashReader, err := hash.NewReader(pipeReader, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
|
||||
hashReader, err := hash.NewReader(pipeReader, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Initialize pipe to stream data to cache
|
||||
rPipe, wPipe := io.Pipe()
|
||||
cHashReader, err := hash.NewReader(rPipe, size, r.MD5HexString(), r.SHA256HexString(), r.ActualSize())
|
||||
cHashReader, err := hash.NewReader(rPipe, size, data.MD5HexString(), data.SHA256HexString(), data.ActualSize())
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
oinfoCh := make(chan ObjectInfo)
|
||||
errCh := make(chan error)
|
||||
go func() {
|
||||
oinfo, perr := putObjectFn(ctx, bucket, object, hashReader, metadata, opts)
|
||||
oinfo, perr := putObjectFn(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts)
|
||||
if perr != nil {
|
||||
pipeWriter.CloseWithError(perr)
|
||||
wPipe.CloseWithError(perr)
|
||||
@@ -695,14 +693,14 @@ func (c cacheObjects) PutObject(ctx context.Context, bucket, object string, r *h
|
||||
}()
|
||||
|
||||
go func() {
|
||||
if err = dcache.Put(ctx, bucket, object, cHashReader, metadata, opts); err != nil {
|
||||
if err = dcache.Put(ctx, bucket, object, NewPutObjReader(cHashReader, nil, nil), metadata, opts); err != nil {
|
||||
wPipe.CloseWithError(err)
|
||||
return
|
||||
}
|
||||
}()
|
||||
|
||||
mwriter := io.MultiWriter(pipeWriter, wPipe)
|
||||
_, err = io.Copy(mwriter, r)
|
||||
_, err = io.Copy(mwriter, data)
|
||||
if err != nil {
|
||||
err = <-errCh
|
||||
return objInfo, err
|
||||
@@ -737,26 +735,27 @@ func (c cacheObjects) NewMultipartUpload(ctx context.Context, bucket, object str
|
||||
}
|
||||
|
||||
// PutObjectPart - uploads part to backend and cache simultaneously.
|
||||
func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
|
||||
func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
|
||||
data := r.Reader
|
||||
putObjectPartFn := c.PutObjectPartFn
|
||||
dcache, err := c.cache.getCacheFS(ctx, bucket, object)
|
||||
if err != nil {
|
||||
// disk cache could not be located,execute backend call.
|
||||
return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
|
||||
}
|
||||
|
||||
if c.isCacheExclude(bucket, object) {
|
||||
return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
|
||||
}
|
||||
|
||||
// make sure cache has at least cacheSizeMultiplier * size available
|
||||
// make sure cache has at least size space available
|
||||
size := data.Size()
|
||||
if !dcache.diskAvailable(size * cacheSizeMultiplier) {
|
||||
if !dcache.diskAvailable(size) {
|
||||
select {
|
||||
case dcache.purgeChan <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
return putObjectPartFn(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
return putObjectPartFn(ctx, bucket, object, uploadID, partID, r, opts)
|
||||
}
|
||||
|
||||
info = PartInfo{}
|
||||
@@ -775,7 +774,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
|
||||
pinfoCh := make(chan PartInfo)
|
||||
errorCh := make(chan error)
|
||||
go func() {
|
||||
info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, hashReader, opts)
|
||||
info, err = putObjectPartFn(ctx, bucket, object, uploadID, partID, NewPutObjReader(hashReader, nil, nil), opts)
|
||||
if err != nil {
|
||||
close(pinfoCh)
|
||||
pipeWriter.CloseWithError(err)
|
||||
@@ -787,7 +786,7 @@ func (c cacheObjects) PutObjectPart(ctx context.Context, bucket, object, uploadI
|
||||
pinfoCh <- info
|
||||
}()
|
||||
go func() {
|
||||
if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, cHashReader, opts); perr != nil {
|
||||
if _, perr := dcache.PutObjectPart(ctx, bucket, object, uploadID, partID, NewPutObjReader(cHashReader, nil, nil), opts); perr != nil {
|
||||
wPipe.CloseWithError(perr)
|
||||
return
|
||||
}
|
||||
@@ -829,25 +828,25 @@ func (c cacheObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - completes multipart upload operation on backend and cache.
|
||||
func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
|
||||
func (c cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
completeMultipartUploadFn := c.CompleteMultipartUploadFn
|
||||
|
||||
if c.isCacheExclude(bucket, object) {
|
||||
return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
|
||||
return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
}
|
||||
|
||||
dcache, err := c.cache.getCacheFS(ctx, bucket, object)
|
||||
if err != nil {
|
||||
// disk cache could not be located,execute backend call.
|
||||
return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
|
||||
return completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
}
|
||||
// perform backend operation
|
||||
objInfo, err = completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts)
|
||||
objInfo, err = completeMultipartUploadFn(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// create new multipart upload in cache with same uploadID
|
||||
dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts)
|
||||
dcache.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -969,7 +968,10 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
|
||||
GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
|
||||
return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
|
||||
},
|
||||
PutObjectFn: func(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
||||
return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
||||
},
|
||||
PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
return newObjectLayerFn().PutObject(ctx, bucket, object, data, metadata, opts)
|
||||
},
|
||||
DeleteObjectFn: func(ctx context.Context, bucket, object string) error {
|
||||
@@ -990,14 +992,14 @@ func newServerCacheObjects(config CacheConfig) (CacheObjectLayer, error) {
|
||||
NewMultipartUploadFn: func(ctx context.Context, bucket, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
|
||||
return newObjectLayerFn().NewMultipartUpload(ctx, bucket, object, metadata, opts)
|
||||
},
|
||||
PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
|
||||
PutObjectPartFn: func(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
|
||||
return newObjectLayerFn().PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
},
|
||||
AbortMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string) error {
|
||||
return newObjectLayerFn().AbortMultipartUpload(ctx, bucket, object, uploadID)
|
||||
},
|
||||
CompleteMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
|
||||
return newObjectLayerFn().CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts)
|
||||
CompleteMultipartUploadFn: func(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
return newObjectLayerFn().CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
},
|
||||
DeleteBucketFn: func(ctx context.Context, bucket string) error {
|
||||
return newObjectLayerFn().DeleteBucket(ctx, bucket)
|
||||
|
||||
@@ -134,7 +134,7 @@ func TestCacheExclusion(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cobj := cobjects.(*cacheObjects)
|
||||
globalServiceDoneCh <- struct{}{}
|
||||
GlobalServiceDoneCh <- struct{}{}
|
||||
testCases := []struct {
|
||||
bucketName string
|
||||
objectName string
|
||||
@@ -199,7 +199,7 @@ func TestDiskCache(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
|
||||
err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -275,12 +275,12 @@ func TestDiskCacheMaxUse(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !cache.diskAvailable(int64(size)) {
|
||||
err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
|
||||
err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
|
||||
if err != errDiskFull {
|
||||
t.Fatal("Cache max-use limit violated.")
|
||||
}
|
||||
} else {
|
||||
err = cache.Put(ctx, bucketName, objectName, hashReader, httpMeta, opts)
|
||||
err = cache.Put(ctx, bucketName, objectName, NewPutObjReader(hashReader, nil, nil), httpMeta, opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -21,7 +21,6 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
@@ -72,7 +71,7 @@ func (api *DummyObjectLayer) GetObjectInfo(ctx context.Context, bucket, object s
|
||||
return
|
||||
}
|
||||
|
||||
func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
func (api *DummyObjectLayer) PutObject(ctx context.Context, bucket, object string, data *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -96,11 +95,11 @@ func (api *DummyObjectLayer) CopyObjectPart(ctx context.Context, srcBucket, srcO
|
||||
return
|
||||
}
|
||||
|
||||
func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (info PartInfo, err error) {
|
||||
func (api *DummyObjectLayer) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
|
||||
func (api *DummyObjectLayer) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -108,7 +107,7 @@ func (api *DummyObjectLayer) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
return
|
||||
}
|
||||
|
||||
func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
|
||||
func (api *DummyObjectLayer) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -155,6 +154,9 @@ func (api *DummyObjectLayer) DeleteBucketPolicy(context.Context, string) (err er
|
||||
func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
|
||||
return
|
||||
}
|
||||
func (api *DummyObjectLayer) IsListenBucketSupported() (b bool) {
|
||||
return
|
||||
}
|
||||
|
||||
func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
|
||||
return
|
||||
|
||||
+274
-80
@@ -22,12 +22,14 @@ import (
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
@@ -37,7 +39,6 @@ import (
|
||||
|
||||
var (
|
||||
// AWS errors for invalid SSE-C requests.
|
||||
errInsecureSSERequest = errors.New("SSE-C requests require TLS connections")
|
||||
errEncryptedObject = errors.New("The object was stored using a form of SSE")
|
||||
errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
|
||||
errKMSNotConfigured = errors.New("KMS not configured for a server side encrypted object")
|
||||
@@ -55,11 +56,11 @@ const (
|
||||
// SSEIVSize is the size of the IV data
|
||||
SSEIVSize = 32 // 32 bytes
|
||||
|
||||
// SSE dare package block size.
|
||||
sseDAREPackageBlockSize = 64 * 1024 // 64KiB bytes
|
||||
// SSEDAREPackageBlockSize - SSE dare package block size.
|
||||
SSEDAREPackageBlockSize = 64 * 1024 // 64KiB bytes
|
||||
|
||||
// SSE dare package meta padding bytes.
|
||||
sseDAREPackageMetaSize = 32 // 32 bytes
|
||||
// SSEDAREPackageMetaSize - SSE dare package meta padding bytes.
|
||||
SSEDAREPackageMetaSize = 32 // 32 bytes
|
||||
|
||||
)
|
||||
|
||||
@@ -80,16 +81,31 @@ func hasServerSideEncryptionHeader(header http.Header) bool {
|
||||
return crypto.S3.IsRequested(header) || crypto.SSEC.IsRequested(header)
|
||||
}
|
||||
|
||||
// isEncryptedMultipart returns true if the current object is
|
||||
// uploaded by the user using multipart mechanism:
|
||||
// initiate new multipart, upload part, complete upload
|
||||
func isEncryptedMultipart(objInfo ObjectInfo) bool {
|
||||
if len(objInfo.Parts) == 0 {
|
||||
return false
|
||||
}
|
||||
if !crypto.IsMultiPart(objInfo.UserDefined) {
|
||||
return false
|
||||
}
|
||||
for _, part := range objInfo.Parts {
|
||||
_, err := sio.DecryptedSize(uint64(part.Size))
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
// Further check if this object is uploaded using multipart mechanism
|
||||
// by the user and it is not about XL internally splitting the
|
||||
// object into parts in PutObject()
|
||||
return !(objInfo.backendType == BackendErasure && len(objInfo.ETag) == 32)
|
||||
}
|
||||
|
||||
// ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request.
|
||||
// It returns the client provided key on success.
|
||||
func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) {
|
||||
if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
|
||||
// we cannot use r.TLS == nil here because Go's http implementation reflects on
|
||||
// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
|
||||
// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
|
||||
// will always fail -> r.TLS is always nil even for TLS requests.
|
||||
return nil, errInsecureSSERequest
|
||||
}
|
||||
if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) {
|
||||
return nil, crypto.ErrIncompatibleEncryptionMethod
|
||||
}
|
||||
@@ -106,13 +122,6 @@ func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) {
|
||||
// ParseSSECustomerHeader parses the SSE-C header fields and returns
|
||||
// the client provided key on success.
|
||||
func ParseSSECustomerHeader(header http.Header) (key []byte, err error) {
|
||||
if !globalIsSSL { // minio only supports HTTP or HTTPS requests not both at the same time
|
||||
// we cannot use r.TLS == nil here because Go's http implementation reflects on
|
||||
// the net.Conn and sets the TLS field of http.Request only if it's an tls.Conn.
|
||||
// Minio uses a BufConn (wrapping a tls.Conn) so the type check within the http package
|
||||
// will always fail -> r.TLS is always nil even for TLS requests.
|
||||
return nil, errInsecureSSERequest
|
||||
}
|
||||
if crypto.S3.IsRequested(header) && crypto.SSEC.IsRequested(header) {
|
||||
return key, crypto.ErrIncompatibleEncryptionMethod
|
||||
}
|
||||
@@ -149,16 +158,40 @@ func rotateKey(oldKey []byte, newKey []byte, bucket, object string, metadata map
|
||||
sealedKey = objectKey.Seal(extKey, sealedKey.IV, crypto.SSEC.String(), bucket, object)
|
||||
crypto.SSEC.CreateMetadata(metadata, sealedKey)
|
||||
return nil
|
||||
case crypto.S3.IsEncrypted(metadata):
|
||||
if GlobalKMS == nil {
|
||||
return errKMSNotConfigured
|
||||
}
|
||||
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
oldKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var objectKey crypto.ObjectKey
|
||||
if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
newKey, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sealedKey = objectKey.Seal(newKey, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object)
|
||||
crypto.S3.CreateMetadata(metadata, globalKMSKeyID, encKey, sealedKey)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]string, sseS3 bool) ([]byte, error) {
|
||||
var sealedKey crypto.SealedKey
|
||||
if sseS3 {
|
||||
if globalKMS == nil {
|
||||
if GlobalKMS == nil {
|
||||
return nil, errKMSNotConfigured
|
||||
}
|
||||
key, encKey, err := globalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
|
||||
key, encKey, err := GlobalKMS.GenerateKey(globalKMSKeyID, crypto.Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -174,21 +207,20 @@ func newEncryptMetadata(key []byte, bucket, object string, metadata map[string]s
|
||||
sealedKey = objectKey.Seal(extKey, crypto.GenerateIV(rand.Reader), crypto.SSEC.String(), bucket, object)
|
||||
crypto.SSEC.CreateMetadata(metadata, sealedKey)
|
||||
return objectKey[:], nil
|
||||
|
||||
}
|
||||
|
||||
func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (io.Reader, error) {
|
||||
func newEncryptReader(content io.Reader, key []byte, bucket, object string, metadata map[string]string, sseS3 bool) (r io.Reader, encKey []byte, err error) {
|
||||
objectEncryptionKey, err := newEncryptMetadata(key, bucket, object, metadata, sseS3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, encKey, err
|
||||
}
|
||||
|
||||
reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20})
|
||||
if err != nil {
|
||||
return nil, crypto.ErrInvalidCustomerKey
|
||||
return nil, encKey, crypto.ErrInvalidCustomerKey
|
||||
}
|
||||
|
||||
return reader, nil
|
||||
return reader, objectEncryptionKey, nil
|
||||
}
|
||||
|
||||
// set new encryption metadata from http request headers for SSE-C and generated key from KMS in the case of
|
||||
@@ -210,19 +242,18 @@ func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[
|
||||
// EncryptRequest takes the client provided content and encrypts the data
|
||||
// with the client provided key. It also marks the object as client-side-encrypted
|
||||
// and sets the correct headers.
|
||||
func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, error) {
|
||||
func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (reader io.Reader, objEncKey []byte, err error) {
|
||||
|
||||
var (
|
||||
key []byte
|
||||
err error
|
||||
)
|
||||
if crypto.S3.IsRequested(r.Header) && crypto.SSEC.IsRequested(r.Header) {
|
||||
return nil, crypto.ErrIncompatibleEncryptionMethod
|
||||
return nil, objEncKey, crypto.ErrIncompatibleEncryptionMethod
|
||||
}
|
||||
if crypto.SSEC.IsRequested(r.Header) {
|
||||
key, err = ParseSSECustomerRequest(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, objEncKey, err
|
||||
}
|
||||
}
|
||||
return newEncryptReader(content, key, bucket, object, metadata, crypto.S3.IsRequested(r.Header))
|
||||
@@ -249,7 +280,7 @@ func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]st
|
||||
default:
|
||||
return nil, errObjectTampered
|
||||
case crypto.S3.IsEncrypted(metadata):
|
||||
if globalKMS == nil {
|
||||
if GlobalKMS == nil {
|
||||
return nil, errKMSNotConfigured
|
||||
}
|
||||
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
|
||||
@@ -257,7 +288,7 @@ func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]st
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
extKey, err := globalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
|
||||
extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -325,7 +356,7 @@ func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket,
|
||||
|
||||
// DecryptCopyRequestR - same as DecryptCopyRequest, but with a
|
||||
// Reader
|
||||
func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, metadata map[string]string) (io.Reader, error) {
|
||||
func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
|
||||
var (
|
||||
key []byte
|
||||
err error
|
||||
@@ -336,7 +367,7 @@ func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string,
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return newDecryptReader(client, key, bucket, object, 0, metadata)
|
||||
return newDecryptReader(client, key, bucket, object, seqNumber, metadata)
|
||||
}
|
||||
|
||||
func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) {
|
||||
@@ -358,17 +389,6 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
// GetEncryptedOffsetLength - returns encrypted offset and length
|
||||
// along with sequence number
|
||||
func GetEncryptedOffsetLength(startOffset, length int64, objInfo ObjectInfo) (seqNumber uint32, encStartOffset, encLength int64) {
|
||||
if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
|
||||
seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
|
||||
return
|
||||
}
|
||||
seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
|
||||
return
|
||||
}
|
||||
|
||||
// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
|
||||
// reader
|
||||
func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
|
||||
@@ -376,13 +396,12 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
|
||||
io.Reader, error) {
|
||||
|
||||
bucket, object := oi.Bucket, oi.Name
|
||||
|
||||
// Single part case
|
||||
if len(oi.Parts) == 0 || !crypto.IsMultiPart(oi.UserDefined) {
|
||||
if !isEncryptedMultipart(oi) {
|
||||
var reader io.Reader
|
||||
var err error
|
||||
if copySource {
|
||||
reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, oi.UserDefined)
|
||||
reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
|
||||
} else {
|
||||
reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined)
|
||||
}
|
||||
@@ -392,8 +411,8 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
|
||||
return reader, nil
|
||||
}
|
||||
|
||||
partDecRelOffset := int64(seqNumber) * sseDAREPackageBlockSize
|
||||
partEncRelOffset := int64(seqNumber) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
|
||||
partDecRelOffset := int64(seqNumber) * SSEDAREPackageBlockSize
|
||||
partEncRelOffset := int64(seqNumber) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
|
||||
|
||||
w := &DecryptBlocksReader{
|
||||
reader: inputReader,
|
||||
@@ -458,7 +477,7 @@ type DecryptBlocksReader struct {
|
||||
// Current part index
|
||||
partIndex int
|
||||
// Parts information
|
||||
parts []objectPartInfo
|
||||
parts []ObjectPartInfo
|
||||
header http.Header
|
||||
bucket, object string
|
||||
metadata map[string]string
|
||||
@@ -558,7 +577,6 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
|
||||
|
||||
d.partDecRelOffset += int64(n1)
|
||||
}
|
||||
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
@@ -574,7 +592,7 @@ type DecryptBlocksWriter struct {
|
||||
// Current part index
|
||||
partIndex int
|
||||
// Parts information
|
||||
parts []objectPartInfo
|
||||
parts []ObjectPartInfo
|
||||
req *http.Request
|
||||
bucket, object string
|
||||
metadata map[string]string
|
||||
@@ -708,7 +726,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
|
||||
var seqNumber uint32
|
||||
var encStartOffset, encLength int64
|
||||
|
||||
if len(objInfo.Parts) == 0 || !crypto.IsMultiPart(objInfo.UserDefined) {
|
||||
if !isEncryptedMultipart(objInfo) {
|
||||
seqNumber, encStartOffset, encLength = getEncryptedSinglePartOffsetLength(startOffset, length, objInfo)
|
||||
|
||||
var writer io.WriteCloser
|
||||
@@ -725,6 +743,7 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
|
||||
}
|
||||
|
||||
seqNumber, encStartOffset, encLength = getEncryptedMultipartsOffsetLength(startOffset, length, objInfo)
|
||||
|
||||
var partStartIndex int
|
||||
var partStartOffset = startOffset
|
||||
// Skip parts until final offset maps to a particular part offset.
|
||||
@@ -747,8 +766,8 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
|
||||
partStartOffset -= int64(decryptedSize)
|
||||
}
|
||||
|
||||
startSeqNum := partStartOffset / sseDAREPackageBlockSize
|
||||
partEncRelOffset := int64(startSeqNum) * (sseDAREPackageBlockSize + sseDAREPackageMetaSize)
|
||||
startSeqNum := partStartOffset / SSEDAREPackageBlockSize
|
||||
partEncRelOffset := int64(startSeqNum) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
|
||||
|
||||
w := &DecryptBlocksWriter{
|
||||
writer: client,
|
||||
@@ -793,7 +812,6 @@ func DecryptBlocksRequest(client io.Writer, r *http.Request, bucket, object stri
|
||||
|
||||
// getEncryptedMultipartsOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
|
||||
func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (uint32, int64, int64) {
|
||||
|
||||
// Calculate encrypted offset of a multipart object
|
||||
computeEncOffset := func(off int64, obj ObjectInfo) (seqNumber uint32, encryptedOffset int64, err error) {
|
||||
var curPartEndOffset uint64
|
||||
@@ -840,9 +858,9 @@ func getEncryptedMultipartsOffsetLength(offset, length int64, obj ObjectInfo) (u
|
||||
|
||||
// getEncryptedSinglePartOffsetLength - fetch sequence number, encrypted start offset and encrypted length.
|
||||
func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo) (seqNumber uint32, encOffset int64, encLength int64) {
|
||||
onePkgSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
|
||||
onePkgSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
|
||||
|
||||
seqNumber = uint32(offset / sseDAREPackageBlockSize)
|
||||
seqNumber = uint32(offset / SSEDAREPackageBlockSize)
|
||||
encOffset = int64(seqNumber) * onePkgSize
|
||||
// The math to compute the encrypted length is always
|
||||
// originalLength i.e (offset+length-1) to be divided under
|
||||
@@ -850,10 +868,10 @@ func getEncryptedSinglePartOffsetLength(offset, length int64, objInfo ObjectInfo
|
||||
// block. This is then multiplied by final package size which
|
||||
// is basically 64KiB + 32. Finally negate the encrypted offset
|
||||
// to get the final encrypted length on disk.
|
||||
encLength = ((offset+length)/sseDAREPackageBlockSize)*onePkgSize - encOffset
|
||||
encLength = ((offset+length)/SSEDAREPackageBlockSize)*onePkgSize - encOffset
|
||||
|
||||
// Check for the remainder, to figure if we need one extract package to read from.
|
||||
if (offset+length)%sseDAREPackageBlockSize > 0 {
|
||||
if (offset+length)%SSEDAREPackageBlockSize > 0 {
|
||||
encLength += onePkgSize
|
||||
}
|
||||
|
||||
@@ -870,7 +888,7 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
|
||||
if !crypto.IsEncrypted(o.UserDefined) {
|
||||
return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
|
||||
}
|
||||
if len(o.Parts) == 0 || !crypto.IsMultiPart(o.UserDefined) {
|
||||
if !isEncryptedMultipart(*o) {
|
||||
size, err := sio.DecryptedSize(uint64(o.Size))
|
||||
if err != nil {
|
||||
err = errObjectTampered // assign correct error type
|
||||
@@ -889,6 +907,64 @@ func (o *ObjectInfo) DecryptedSize() (int64, error) {
|
||||
return size, nil
|
||||
}
|
||||
|
||||
// For encrypted objects, the ETag sent by client if available
|
||||
// is stored in encrypted form in the backend. Decrypt the ETag
|
||||
// if ETag was previously encrypted.
|
||||
func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool) (decryptedETag string) {
|
||||
var (
|
||||
key [32]byte
|
||||
err error
|
||||
)
|
||||
// If ETag is contentMD5Sum return it as is.
|
||||
if len(objInfo.ETag) == 32 {
|
||||
return objInfo.ETag
|
||||
}
|
||||
|
||||
if crypto.IsMultiPart(objInfo.UserDefined) {
|
||||
return objInfo.ETag
|
||||
}
|
||||
if crypto.SSECopy.IsRequested(headers) {
|
||||
key, err = crypto.SSECopy.ParseHTTP(headers)
|
||||
if err != nil {
|
||||
return objInfo.ETag
|
||||
}
|
||||
}
|
||||
// As per AWS S3 Spec, ETag for SSE-C encrypted objects need not be MD5Sum of the data.
|
||||
// Since server side copy with same source and dest just replaces the ETag, we save
|
||||
// encrypted content MD5Sum as ETag for both SSE-C and SSE-S3, we standardize the ETag
|
||||
//encryption across SSE-C and SSE-S3, and only return last 32 bytes for SSE-C
|
||||
if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && !copySource {
|
||||
return objInfo.ETag[len(objInfo.ETag)-32:]
|
||||
}
|
||||
|
||||
objectEncryptionKey, err := decryptObjectInfo(key[:], objInfo.Bucket, objInfo.Name, objInfo.UserDefined)
|
||||
if err != nil {
|
||||
return objInfo.ETag
|
||||
}
|
||||
return tryDecryptETag(objectEncryptionKey, objInfo.ETag, false)
|
||||
}
|
||||
|
||||
// helper to decrypt Etag given object encryption key and encrypted ETag
|
||||
func tryDecryptETag(key []byte, encryptedETag string, ssec bool) string {
|
||||
// ETag for SSE-C encrypted objects need not be content MD5Sum.While encrypted
|
||||
// md5sum is stored internally, return just the last 32 bytes of hex-encoded and
|
||||
// encrypted md5sum string for SSE-C
|
||||
if ssec {
|
||||
return encryptedETag[len(encryptedETag)-32:]
|
||||
}
|
||||
var objectKey crypto.ObjectKey
|
||||
copy(objectKey[:], key)
|
||||
encBytes, err := hex.DecodeString(encryptedETag)
|
||||
if err != nil {
|
||||
return encryptedETag
|
||||
}
|
||||
etagBytes, err := objectKey.UnsealETag(encBytes)
|
||||
if err != nil {
|
||||
return encryptedETag
|
||||
}
|
||||
return hex.EncodeToString(etagBytes)
|
||||
}
|
||||
|
||||
// GetDecryptedRange - To decrypt the range (off, length) of the
|
||||
// decrypted object stream, we need to read the range (encOff,
|
||||
// encLength) of the encrypted object stream to decrypt it, and
|
||||
@@ -910,26 +986,29 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
|
||||
}
|
||||
|
||||
// Assemble slice of (decrypted) part sizes in `sizes`
|
||||
var sizes []int64
|
||||
var decObjSize int64 // decrypted total object size
|
||||
var partSize uint64
|
||||
partSize, err = sio.DecryptedSize(uint64(o.Size))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
sizes := []int64{int64(partSize)}
|
||||
decObjSize = sizes[0]
|
||||
if crypto.IsMultiPart(o.UserDefined) {
|
||||
if isEncryptedMultipart(*o) {
|
||||
sizes = make([]int64, len(o.Parts))
|
||||
decObjSize = 0
|
||||
for i, part := range o.Parts {
|
||||
var partSize uint64
|
||||
partSize, err = sio.DecryptedSize(uint64(part.Size))
|
||||
if err != nil {
|
||||
err = errObjectTampered
|
||||
return
|
||||
}
|
||||
t := int64(partSize)
|
||||
sizes[i] = t
|
||||
decObjSize += t
|
||||
sizes[i] = int64(partSize)
|
||||
decObjSize += int64(partSize)
|
||||
}
|
||||
} else {
|
||||
var partSize uint64
|
||||
partSize, err = sio.DecryptedSize(uint64(o.Size))
|
||||
if err != nil {
|
||||
err = errObjectTampered
|
||||
return
|
||||
}
|
||||
sizes = []int64{int64(partSize)}
|
||||
decObjSize = sizes[0]
|
||||
}
|
||||
|
||||
var off, length int64
|
||||
@@ -963,11 +1042,11 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
|
||||
// partStart is always found in the loop above,
|
||||
// because off is validated.
|
||||
|
||||
sseDAREEncPackageBlockSize := int64(sseDAREPackageBlockSize + sseDAREPackageMetaSize)
|
||||
startPkgNum := (off - cumulativeSum) / sseDAREPackageBlockSize
|
||||
sseDAREEncPackageBlockSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
|
||||
startPkgNum := (off - cumulativeSum) / SSEDAREPackageBlockSize
|
||||
|
||||
// Now we can calculate the number of bytes to skip
|
||||
skipLen = (off - cumulativeSum) % sseDAREPackageBlockSize
|
||||
skipLen = (off - cumulativeSum) % SSEDAREPackageBlockSize
|
||||
|
||||
encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize
|
||||
// Locate the part containing the end of the required range
|
||||
@@ -984,7 +1063,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
|
||||
}
|
||||
// partEnd is always found in the loop above, because off and
|
||||
// length are validated.
|
||||
endPkgNum := (endOffset - cumulativeSum) / sseDAREPackageBlockSize
|
||||
endPkgNum := (endOffset - cumulativeSum) / SSEDAREPackageBlockSize
|
||||
// Compute endEncOffset with one additional DARE package (so
|
||||
// we read the package containing the last desired byte).
|
||||
endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize
|
||||
@@ -1039,7 +1118,7 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
|
||||
}
|
||||
var err error
|
||||
if info.Size, err = info.DecryptedSize(); err != nil {
|
||||
apiErr = toAPIErrorCode(err)
|
||||
apiErr = toAPIErrorCode(context.Background(), err)
|
||||
}
|
||||
}
|
||||
return
|
||||
@@ -1052,7 +1131,7 @@ func DecryptCopyObjectInfo(info *ObjectInfo, headers http.Header) (apiErr APIErr
|
||||
// decryption succeeded.
|
||||
//
|
||||
// DecryptObjectInfo also returns whether the object is encrypted or not.
|
||||
func DecryptObjectInfo(info ObjectInfo, headers http.Header) (encrypted bool, err error) {
|
||||
func DecryptObjectInfo(info *ObjectInfo, headers http.Header) (encrypted bool, err error) {
|
||||
// Directories are never encrypted.
|
||||
if info.IsDir {
|
||||
return false, nil
|
||||
@@ -1071,6 +1150,121 @@ func DecryptObjectInfo(info ObjectInfo, headers http.Header) (encrypted bool, er
|
||||
return
|
||||
}
|
||||
_, err = info.DecryptedSize()
|
||||
|
||||
if crypto.IsEncrypted(info.UserDefined) && !crypto.IsMultiPart(info.UserDefined) {
|
||||
info.ETag = getDecryptedETag(headers, *info, false)
|
||||
}
|
||||
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// The customer key in the header is used by the gateway for encryption in the case of
|
||||
// s3 gateway double encryption. A new client key is derived from the customer provided
|
||||
// key to be sent to the s3 backend for encryption at the backend.
|
||||
func deriveClientKey(clientKey [32]byte, bucket, object string) [32]byte {
|
||||
var key [32]byte
|
||||
mac := hmac.New(sha256.New, clientKey[:])
|
||||
mac.Write([]byte(crypto.SSEC.String()))
|
||||
mac.Write([]byte(path.Join(bucket, object)))
|
||||
mac.Sum(key[:0])
|
||||
return key
|
||||
}
|
||||
|
||||
// extract encryption options for pass through to backend in the case of gateway
|
||||
func extractEncryptionOption(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
|
||||
var clientKey [32]byte
|
||||
var sse encrypt.ServerSide
|
||||
|
||||
if copySource {
|
||||
if crypto.SSECopy.IsRequested(header) {
|
||||
clientKey, err = crypto.SSECopy.ParseHTTP(header)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
|
||||
return
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(sse)}, nil
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if crypto.SSEC.IsRequested(header) {
|
||||
clientKey, err = crypto.SSEC.ParseHTTP(header)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil {
|
||||
return
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: sse}, nil
|
||||
}
|
||||
if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) {
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
// get ObjectOptions for GET calls from encryption headers
|
||||
func getEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
|
||||
var (
|
||||
encryption encrypt.ServerSide
|
||||
opts ObjectOptions
|
||||
)
|
||||
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
|
||||
key, err := crypto.SSEC.ParseHTTP(r.Header)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
derivedKey := deriveClientKey(key, bucket, object)
|
||||
encryption, err = encrypt.NewSSEC(derivedKey[:])
|
||||
logger.CriticalIf(ctx, err)
|
||||
return ObjectOptions{ServerSideEncryption: encryption}, nil
|
||||
}
|
||||
// default case of passing encryption headers to backend
|
||||
return extractEncryptionOption(r.Header, false, nil)
|
||||
}
|
||||
|
||||
// get ObjectOptions for PUT calls from encryption headers
|
||||
func putEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
|
||||
// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
|
||||
// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
|
||||
if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.NewSSE()}, nil
|
||||
}
|
||||
if GlobalGatewaySSE.SSEC() && crypto.SSEC.IsRequested(r.Header) {
|
||||
return getEncryptionOpts(ctx, r, bucket, object)
|
||||
}
|
||||
// default case of passing encryption headers to backend
|
||||
return extractEncryptionOption(r.Header, false, metadata)
|
||||
}
|
||||
|
||||
// get ObjectOptions for Copy calls for encryption headers provided on the target side
|
||||
func copyDstEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) {
|
||||
return putEncryptionOpts(ctx, r, bucket, object, metadata)
|
||||
}
|
||||
|
||||
// get ObjectOptions for Copy calls for encryption headers provided on the source side
|
||||
func copySrcEncryptionOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) {
|
||||
var (
|
||||
ssec encrypt.ServerSide
|
||||
opts ObjectOptions
|
||||
)
|
||||
|
||||
if GlobalGatewaySSE.SSEC() && crypto.SSECopy.IsRequested(r.Header) {
|
||||
key, err := crypto.SSECopy.ParseHTTP(r.Header)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
derivedKey := deriveClientKey(key, bucket, object)
|
||||
ssec, err = encrypt.NewSSEC(derivedKey[:])
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
return ObjectOptions{ServerSideEncryption: encrypt.SSECopy(ssec)}, nil
|
||||
}
|
||||
|
||||
// default case of passing encryption headers to backend
|
||||
return extractEncryptionOption(r.Header, true, nil)
|
||||
}
|
||||
|
||||
+148
-228
@@ -23,6 +23,7 @@ import (
|
||||
"testing"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
@@ -38,7 +39,7 @@ var hasServerSideEncryptionHeaderTests = []struct {
|
||||
{headers: map[string]string{}, sseRequest: false}, // 4
|
||||
{headers: map[string]string{crypto.SSECopyAlgorithm + " ": "AES256", " " + crypto.SSECopyKey: "key", crypto.SSECopyKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
|
||||
{headers: map[string]string{crypto.SSECopyAlgorithm: "", crypto.SSECopyKey: "", crypto.SSECopyKeyMD5: ""}, sseRequest: false}, // 6
|
||||
{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: true}, // 6
|
||||
{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: true}, // 7
|
||||
}
|
||||
|
||||
func TestHasServerSideEncryptionHeader(t *testing.T) {
|
||||
@@ -90,12 +91,12 @@ var hasSSECustomerHeaderTests = []struct {
|
||||
{headers: map[string]string{crypto.SSECKeyMD5: "md5"}, sseRequest: true}, // 3
|
||||
{headers: map[string]string{}, sseRequest: false}, // 4
|
||||
{headers: map[string]string{crypto.SSECAlgorithm + " ": "AES256", " " + crypto.SSECKey: "key", crypto.SSECKeyMD5 + " ": "md5"}, sseRequest: false}, // 5
|
||||
{headers: map[string]string{crypto.SSECAlgorithm: "", crypto.SSECKey: "", crypto.SSECKeyMD5: ""}, sseRequest: false}, // 6
|
||||
{headers: map[string]string{crypto.SSECAlgorithm: "", crypto.SSECKey: "", crypto.SSECKeyMD5: ""}, sseRequest: true}, // 6
|
||||
{headers: map[string]string{crypto.SSEHeader: ""}, sseRequest: false}, // 7
|
||||
|
||||
}
|
||||
|
||||
func TesthasSSECustomerHeader(t *testing.T) {
|
||||
func TestHasSSECustomerHeader(t *testing.T) {
|
||||
for i, test := range hasSSECustomerHeaderTests {
|
||||
headers := http.Header{}
|
||||
for k, v := range test.headers {
|
||||
@@ -107,228 +108,6 @@ func TesthasSSECustomerHeader(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
var parseSSECustomerRequestTests = []struct {
|
||||
headers map[string]string
|
||||
useTLS bool
|
||||
err error
|
||||
}{
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
|
||||
crypto.SSECKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
useTLS: true, err: nil,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
|
||||
crypto.SSECKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
useTLS: false, err: errInsecureSSERequest,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES 256",
|
||||
crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
|
||||
crypto.SSECKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrInvalidCustomerAlgorithm,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
|
||||
crypto.SSECKeyMD5: "H+jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
|
||||
crypto.SSECKeyMD5: "H+jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrInvalidCustomerKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
|
||||
crypto.SSECKeyMD5: " +jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
|
||||
crypto.SSECKeyMD5: "9KPgDdZNTHimuYCwnJTp5g==",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrInvalidCustomerKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "", // 7
|
||||
crypto.SSECKeyMD5: "9KPgDdZNTHimuYCwnJTp5g==",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrMissingCustomerKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
|
||||
crypto.SSECKeyMD5: "",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrMissingCustomerKeyMD5,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
|
||||
crypto.SSECKeyMD5: "",
|
||||
crypto.SSEHeader: "",
|
||||
},
|
||||
useTLS: true, err: crypto.ErrIncompatibleEncryptionMethod,
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseSSECustomerRequest(t *testing.T) {
|
||||
defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
|
||||
for i, test := range parseSSECustomerRequestTests {
|
||||
headers := http.Header{}
|
||||
for k, v := range test.headers {
|
||||
headers.Set(k, v)
|
||||
}
|
||||
request := &http.Request{}
|
||||
request.Header = headers
|
||||
globalIsSSL = test.useTLS
|
||||
|
||||
_, err := ParseSSECustomerRequest(request)
|
||||
if err != test.err {
|
||||
t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var parseSSECopyCustomerRequestTests = []struct {
|
||||
headers map[string]string
|
||||
metadata map[string]string
|
||||
useTLS bool
|
||||
err error
|
||||
}{
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
|
||||
crypto.SSECopyKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: nil,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 0
|
||||
crypto.SSECopyKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": base64.StdEncoding.EncodeToString(make([]byte, 64))},
|
||||
useTLS: true, err: crypto.ErrIncompatibleEncryptionMethod,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 1
|
||||
crypto.SSECopyKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: false, err: errInsecureSSERequest,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES 256",
|
||||
crypto.SSECopyKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", // 2
|
||||
crypto.SSECopyKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: crypto.ErrInvalidCustomerAlgorithm,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 3
|
||||
crypto.SSECopyKeyMD5: "H+jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: " jE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 4
|
||||
crypto.SSECopyKeyMD5: "H+jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: crypto.ErrInvalidCustomerKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "NjE0SL87s+ZhYtaTrg5eI5cjhCQLGPVMKenPG2bCJFw=", // 5
|
||||
crypto.SSECopyKeyMD5: " +jq/LwEOEO90YtiTuNFVw==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: crypto.ErrCustomerKeyMD5Mismatch,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 6
|
||||
crypto.SSECopyKeyMD5: "9KPgDdZNTHimuYCwnJTp5g==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: crypto.ErrInvalidCustomerKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "", // 7
|
||||
crypto.SSECopyKeyMD5: "9KPgDdZNTHimuYCwnJTp5g==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: crypto.ErrMissingCustomerKey,
|
||||
},
|
||||
{
|
||||
headers: map[string]string{
|
||||
crypto.SSECopyAlgorithm: "AES256",
|
||||
crypto.SSECopyKey: "vFQ9ScFOF6Tu/BfzMS+rVMvlZGJHi5HmGJenJfrfKI45", // 8
|
||||
crypto.SSECopyKeyMD5: "",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
useTLS: true, err: crypto.ErrMissingCustomerKeyMD5,
|
||||
},
|
||||
}
|
||||
|
||||
func TestParseSSECopyCustomerRequest(t *testing.T) {
|
||||
defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
|
||||
for i, test := range parseSSECopyCustomerRequestTests {
|
||||
headers := http.Header{}
|
||||
for k, v := range test.headers {
|
||||
headers.Set(k, v)
|
||||
}
|
||||
request := &http.Request{}
|
||||
request.Header = headers
|
||||
globalIsSSL = test.useTLS
|
||||
|
||||
_, err := ParseSSECopyCustomerRequest(request.Header, test.metadata)
|
||||
if err != test.err {
|
||||
t.Errorf("Test %d: Parse returned: %v want: %v", i, err, test.err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var encryptRequestTests = []struct {
|
||||
header map[string]string
|
||||
metadata map[string]string
|
||||
@@ -362,7 +141,7 @@ func TestEncryptRequest(t *testing.T) {
|
||||
for k, v := range test.header {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
_, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
|
||||
_, _, err := EncryptRequest(content, req, "bucket", "object", test.metadata)
|
||||
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
|
||||
@@ -547,7 +326,7 @@ var decryptObjectInfoTests = []struct {
|
||||
|
||||
func TestDecryptObjectInfo(t *testing.T) {
|
||||
for i, test := range decryptObjectInfoTests {
|
||||
if encrypted, err := DecryptObjectInfo(test.info, test.headers); err != test.expErr {
|
||||
if encrypted, err := DecryptObjectInfo(&test.info, test.headers); err != test.expErr {
|
||||
t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
|
||||
} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
|
||||
t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
|
||||
@@ -557,6 +336,66 @@ func TestDecryptObjectInfo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Tests for issue reproduced when getting the right encrypted
|
||||
// offset of the object.
|
||||
func TestGetDecryptedRange_Issue50(t *testing.T) {
|
||||
rs, err := parseRequestRangeSpec("bytes=594870256-594870263")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
objInfo := ObjectInfo{
|
||||
Bucket: "bucket",
|
||||
Name: "object",
|
||||
Size: 595160760,
|
||||
UserDefined: map[string]string{
|
||||
crypto.SSEMultipart: "",
|
||||
crypto.SSEIV: "HTexa=",
|
||||
crypto.SSESealAlgorithm: "DAREv2-HMAC-SHA256",
|
||||
crypto.SSECSealedKey: "IAA8PGAA==",
|
||||
ReservedMetadataPrefix + "actual-size": "594870264",
|
||||
"content-type": "application/octet-stream",
|
||||
"etag": "166b1545b4c1535294ee0686678bea8c-2",
|
||||
},
|
||||
Parts: []ObjectPartInfo{
|
||||
{
|
||||
Number: 1,
|
||||
Name: "part.1",
|
||||
ETag: "etag1",
|
||||
Size: 297580380,
|
||||
ActualSize: 297435132,
|
||||
},
|
||||
{
|
||||
Number: 2,
|
||||
Name: "part.2",
|
||||
ETag: "etag2",
|
||||
Size: 297580380,
|
||||
ActualSize: 297435132,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
encOff, encLength, skipLen, seqNumber, partStart, err := objInfo.GetDecryptedRange(rs)
|
||||
if err != nil {
|
||||
t.Fatalf("Test: failed %s", err)
|
||||
}
|
||||
if encOff != 595127964 {
|
||||
t.Fatalf("Test: expected %d, got %d", 595127964, encOff)
|
||||
}
|
||||
if encLength != 32796 {
|
||||
t.Fatalf("Test: expected %d, got %d", 32796, encLength)
|
||||
}
|
||||
if skipLen != 32756 {
|
||||
t.Fatalf("Test: expected %d, got %d", 32756, skipLen)
|
||||
}
|
||||
if seqNumber != 4538 {
|
||||
t.Fatalf("Test: expected %d, got %d", 4538, seqNumber)
|
||||
}
|
||||
if partStart != 1 {
|
||||
t.Fatalf("Test: expected %d, got %d", 1, partStart)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetDecryptedRange(t *testing.T) {
|
||||
var (
|
||||
pkgSz = int64(64) * humanize.KiByte
|
||||
@@ -666,7 +505,7 @@ func TestGetDecryptedRange(t *testing.T) {
|
||||
var (
|
||||
// make a multipart object-info given part sizes
|
||||
mkMPObj = func(sizes []int64) ObjectInfo {
|
||||
r := make([]objectPartInfo, len(sizes))
|
||||
r := make([]ObjectPartInfo, len(sizes))
|
||||
sum := int64(0)
|
||||
for i, s := range sizes {
|
||||
r[i].Number = i
|
||||
@@ -838,3 +677,84 @@ func TestGetDecryptedRange(t *testing.T) {
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
var extractEncryptionOptionTests = []struct {
|
||||
headers http.Header
|
||||
copySource bool
|
||||
metadata map[string]string
|
||||
encryptionType encrypt.Type
|
||||
err error
|
||||
}{
|
||||
{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
|
||||
crypto.SSECKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: encrypt.SSEC,
|
||||
err: nil}, // 0
|
||||
{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
|
||||
crypto.SSECKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: true,
|
||||
metadata: nil,
|
||||
encryptionType: "",
|
||||
err: nil}, // 1
|
||||
{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
|
||||
crypto.SSECKey: []string{"Mz"},
|
||||
crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: "",
|
||||
err: crypto.ErrInvalidCustomerKey}, // 2
|
||||
{headers: http.Header{crypto.SSEHeader: []string{"AES256"}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: encrypt.S3,
|
||||
err: nil}, // 3
|
||||
{headers: http.Header{},
|
||||
copySource: false,
|
||||
metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
crypto.S3KMSKeyID: "kms-key",
|
||||
crypto.S3KMSSealedKey: "m-key"},
|
||||
encryptionType: encrypt.S3,
|
||||
err: nil}, // 4
|
||||
{headers: http.Header{},
|
||||
copySource: true,
|
||||
metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
crypto.S3KMSKeyID: "kms-key",
|
||||
crypto.S3KMSSealedKey: "m-key"},
|
||||
encryptionType: "",
|
||||
err: nil}, // 5
|
||||
{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
|
||||
crypto.SSECopyKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: true,
|
||||
metadata: nil,
|
||||
encryptionType: encrypt.SSEC,
|
||||
err: nil}, // 6
|
||||
{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
|
||||
crypto.SSECopyKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: "",
|
||||
err: nil}, // 7
|
||||
}
|
||||
|
||||
func TestExtractEncryptionOptions(t *testing.T) {
|
||||
for i, test := range extractEncryptionOptionTests {
|
||||
opts, err := extractEncryptionOption(test.headers, test.copySource, test.metadata)
|
||||
if test.err != err {
|
||||
t.Errorf("Case %d: expected err: %v , actual err: %v", i, test.err, err)
|
||||
}
|
||||
if err == nil {
|
||||
if opts.ServerSideEncryption == nil && test.encryptionType != "" {
|
||||
t.Errorf("Case %d: expected opts to be of %v encryption type", i, test.encryptionType)
|
||||
|
||||
}
|
||||
if opts.ServerSideEncryption != nil && test.encryptionType != opts.ServerSideEncryption.Type() {
|
||||
t.Errorf("Case %d: expected opts to have encryption type %v but was %v ", i, test.encryptionType, opts.ServerSideEncryption.Type())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+102
-45
@@ -226,6 +226,17 @@ func TestGetSetIndexes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func getHexSequences(start int, number int, paddinglen int) (seq []string) {
|
||||
for i := start; i <= number; i++ {
|
||||
if paddinglen == 0 {
|
||||
seq = append(seq, fmt.Sprintf("%x", i))
|
||||
} else {
|
||||
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", paddinglen), i))
|
||||
}
|
||||
}
|
||||
return seq
|
||||
}
|
||||
|
||||
func getSequences(start int, number int, paddinglen int) (seq []string) {
|
||||
for i := start; i <= number; i++ {
|
||||
if paddinglen == 0 {
|
||||
@@ -287,9 +298,9 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"/export/set",
|
||||
"",
|
||||
getSequences(1, 64, 0),
|
||||
Prefix: "/export/set",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 64, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -305,14 +316,14 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(1, 64, 0),
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 64, 0),
|
||||
},
|
||||
{
|
||||
"http://minio",
|
||||
"/export/set",
|
||||
getSequences(2, 3, 0),
|
||||
Prefix: "http://minio",
|
||||
Suffix: "/export/set",
|
||||
Seq: getSequences(2, 3, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -328,9 +339,9 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"http://minio",
|
||||
".mydomain.net/data",
|
||||
getSequences(1, 64, 0),
|
||||
Prefix: "http://minio",
|
||||
Suffix: ".mydomain.net/data",
|
||||
Seq: getSequences(1, 64, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -345,14 +356,14 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"/data",
|
||||
getSequences(1, 16, 0),
|
||||
Prefix: "",
|
||||
Suffix: "/data",
|
||||
Seq: getSequences(1, 16, 0),
|
||||
},
|
||||
{
|
||||
"http://rack",
|
||||
".mydomain.minio",
|
||||
getSequences(1, 4, 0),
|
||||
Prefix: "http://rack",
|
||||
Suffix: ".mydomain.minio",
|
||||
Seq: getSequences(1, 4, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -368,14 +379,14 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(0, 1, 0),
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(0, 1, 0),
|
||||
},
|
||||
{
|
||||
"http://minio",
|
||||
".mydomain.net/data",
|
||||
getSequences(0, 15, 0),
|
||||
Prefix: "http://minio",
|
||||
Suffix: ".mydomain.net/data",
|
||||
Seq: getSequences(0, 15, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -391,9 +402,9 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"http://server1/data",
|
||||
"",
|
||||
getSequences(1, 32, 0),
|
||||
Prefix: "http://server1/data",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 32, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -409,9 +420,9 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"http://server1/data",
|
||||
"",
|
||||
getSequences(1, 32, 2),
|
||||
Prefix: "http://server1/data",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 32, 2),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -427,19 +438,19 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(1, 2, 0),
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 2, 0),
|
||||
},
|
||||
{
|
||||
"",
|
||||
"/test",
|
||||
getSequences(1, 64, 0),
|
||||
Prefix: "",
|
||||
Suffix: "/test",
|
||||
Seq: getSequences(1, 64, 0),
|
||||
},
|
||||
{
|
||||
"http://minio",
|
||||
"/export/set",
|
||||
getSequences(2, 3, 0),
|
||||
Prefix: "http://minio",
|
||||
Suffix: "/export/set",
|
||||
Seq: getSequences(2, 3, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
@@ -456,14 +467,60 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
"",
|
||||
"",
|
||||
getSequences(1, 10, 0),
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
"/export",
|
||||
"/disk",
|
||||
getSequences(1, 10, 0),
|
||||
Prefix: "/export",
|
||||
Suffix: "/disk",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// IPv6 ellipses with hexadecimal expansion
|
||||
{
|
||||
"http://[2001:3984:3989::{1...a}]/disk{1...10}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
Prefix: "http://[2001:3984:3989::",
|
||||
Suffix: "]/disk",
|
||||
Seq: getHexSequences(1, 10, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// IPv6 ellipses with hexadecimal expansion with 3 position numerics.
|
||||
{
|
||||
"http://[2001:3984:3989::{001...00a}]/disk{1...10}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
Prefix: "http://[2001:3984:3989::",
|
||||
Suffix: "]/disk",
|
||||
Seq: getHexSequences(1, 10, 3),
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
+59
-23
@@ -29,6 +29,7 @@ import (
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/disk"
|
||||
"github.com/minio/minio/pkg/mountinfo"
|
||||
)
|
||||
|
||||
@@ -112,6 +113,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
|
||||
}
|
||||
}
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
|
||||
if host == "" {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
|
||||
@@ -152,7 +156,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
// Only check if the arg is an ip address and ask for scheme since its absent.
|
||||
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
|
||||
// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
|
||||
if isHostIPv4(arg) {
|
||||
if isHostIP(arg) {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
|
||||
}
|
||||
u = &url.URL{Path: path.Clean(arg)}
|
||||
@@ -194,6 +198,34 @@ func (endpoints EndpointList) GetString(i int) string {
|
||||
return endpoints[i].String()
|
||||
}
|
||||
|
||||
// localEndpointsPerf - returns ServerDrivesPerfInfo for only the
|
||||
// local endpoints from given list of endpoints
|
||||
func localEndpointsPerf(endpoints EndpointList) ServerDrivesPerfInfo {
|
||||
var dps []disk.Performance
|
||||
var addr string
|
||||
for _, endpoint := range endpoints {
|
||||
// Only proceed for local endpoints
|
||||
if endpoint.IsLocal {
|
||||
addr = GetLocalPeer(endpoints)
|
||||
if _, err := os.Stat(endpoint.Path); err != nil {
|
||||
// Since this drive is not available, add relevant details and proceed
|
||||
dps = append(dps, disk.Performance{Path: endpoint.Path, Error: err.Error()})
|
||||
continue
|
||||
}
|
||||
tempObj := mustGetUUID()
|
||||
fsPath := pathJoin(endpoint.Path, minioMetaTmpBucket, tempObj)
|
||||
dp := disk.GetPerformance(fsPath)
|
||||
dp.Path = endpoint.Path
|
||||
dps = append(dps, dp)
|
||||
}
|
||||
}
|
||||
|
||||
return ServerDrivesPerfInfo{
|
||||
Addr: addr,
|
||||
Perf: dps,
|
||||
}
|
||||
}
|
||||
|
||||
// NewEndpointList - returns new endpoint list based on input args.
|
||||
func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
|
||||
var endpointType EndpointType
|
||||
@@ -224,7 +256,6 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
|
||||
uniqueArgs.Add(arg)
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
@@ -341,7 +372,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
if err != nil {
|
||||
host = endpoint.Host
|
||||
}
|
||||
hostIPSet, _ := getHostIP4(host)
|
||||
hostIPSet, _ := getHostIP(host)
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return serverAddr, endpoints, setupType,
|
||||
@@ -411,12 +442,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
host = localServerAddr
|
||||
}
|
||||
|
||||
ipList, err := getHostIP4(host)
|
||||
ipList, err := getHostIP(host)
|
||||
logger.FatalIf(err, "unexpected error when resolving host '%s'", host)
|
||||
|
||||
// Filter ipList by IPs those start with '127.'.
|
||||
// Filter ipList by IPs those start with '127.' or '::1'
|
||||
loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return strings.HasPrefix(ip, "127.")
|
||||
return strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
|
||||
}, "")
|
||||
|
||||
// If loop back IP is found and ipList contains only loop back IPs, then error out.
|
||||
@@ -455,7 +486,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
updateDomainIPs(uniqueArgs)
|
||||
_, dok := os.LookupEnv("MINIO_DOMAIN")
|
||||
_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
|
||||
_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if dok && eok && !iok {
|
||||
updateDomainIPs(uniqueArgs)
|
||||
}
|
||||
|
||||
setupType = DistXLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
@@ -480,10 +516,10 @@ func GetLocalPeer(endpoints EndpointList) (localPeer string) {
|
||||
// Local peer can be empty in FS or Erasure coded mode.
|
||||
// If so, return globalMinioHost + globalMinioPort value.
|
||||
if globalMinioHost != "" {
|
||||
return globalMinioHost + ":" + globalMinioPort
|
||||
return net.JoinHostPort(globalMinioHost, globalMinioPort)
|
||||
}
|
||||
|
||||
return "127.0.0.1:" + globalMinioPort
|
||||
return net.JoinHostPort("127.0.0.1", globalMinioPort)
|
||||
}
|
||||
return peerSet.ToSlice()[0]
|
||||
}
|
||||
@@ -509,21 +545,21 @@ func GetRemotePeers(endpoints EndpointList) []string {
|
||||
return peerSet.ToSlice()
|
||||
}
|
||||
|
||||
// In federated and distributed setup, update IP addresses of the hosts passed in command line
|
||||
// if MINIO_PUBLIC_IPS are not set manually
|
||||
func updateDomainIPs(endPoints set.StringSet) {
|
||||
_, dok := os.LookupEnv("MINIO_DOMAIN")
|
||||
_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
|
||||
_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if dok && eok && !iok {
|
||||
globalDomainIPs = set.NewStringSet()
|
||||
for e := range endPoints {
|
||||
host, _, _ := net.SplitHostPort(e)
|
||||
ipList, _ := getHostIP4(host)
|
||||
remoteIPList := ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.")
|
||||
}, "")
|
||||
globalDomainIPs.Add(remoteIPList.ToSlice()[0])
|
||||
ipList := set.NewStringSet()
|
||||
for e := range endPoints {
|
||||
host, _, err := net.SplitHostPort(e)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "missing port in address") {
|
||||
host = e
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
IPs, _ := getHostIP(host)
|
||||
ipList = ipList.Union(IPs)
|
||||
}
|
||||
globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
|
||||
}, "")
|
||||
}
|
||||
|
||||
@@ -304,27 +304,27 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
}, DistXLSetupType, nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
for i, testCase := range testCases {
|
||||
serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
|
||||
|
||||
if err == nil {
|
||||
if testCase.expectedErr != nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
t.Fatalf("Test (%d) error: expected = %v, got = <nil>", i+1, testCase.expectedErr)
|
||||
} else {
|
||||
if serverAddr != testCase.expectedServerAddr {
|
||||
t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
|
||||
t.Fatalf("Test (%d) serverAddr: expected = %v, got = %v", i+1, testCase.expectedServerAddr, serverAddr)
|
||||
}
|
||||
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
|
||||
t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
|
||||
t.Fatalf("Test (%d) endpoints: expected = %v, got = %v", i+1, testCase.expectedEndpoints, endpoints)
|
||||
}
|
||||
if setupType != testCase.expectedSetupType {
|
||||
t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
|
||||
t.Fatalf("Test (%d) setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
|
||||
}
|
||||
}
|
||||
} else if testCase.expectedErr == nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
t.Fatalf("Test (%d) error: expected = <nil>, got = %v", i+1, err)
|
||||
} else if err.Error() != testCase.expectedErr.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
t.Fatalf("Test (%d) error: expected = %v, got = %v", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
// Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvKMSMasterKey is the environment variable used to specify
|
||||
// a KMS master key used to protect SSE-S3 per-object keys.
|
||||
// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
|
||||
EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
|
||||
|
||||
// EnvAutoEncryption is the environment variable used to en/disable
|
||||
// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
|
||||
// requires a valid KMS configuration and turns any non-SSE-C
|
||||
// request into an SSE-S3 request.
|
||||
// If present EnvAutoEncryption must be either "on" or "off".
|
||||
EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvVaultEndpoint is the environment variable used to specify
|
||||
// the vault HTTPS endpoint.
|
||||
EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
|
||||
|
||||
// EnvVaultAuthType is the environment variable used to specify
|
||||
// the authentication type for vault.
|
||||
EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
|
||||
|
||||
// EnvVaultAppRoleID is the environment variable used to specify
|
||||
// the vault AppRole ID.
|
||||
EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
|
||||
|
||||
// EnvVaultAppSecretID is the environment variable used to specify
|
||||
// the vault AppRole secret corresponding to the AppRole ID.
|
||||
EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
|
||||
|
||||
// EnvVaultKeyVersion is the environment variable used to specify
|
||||
// the vault key version.
|
||||
EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
|
||||
|
||||
// EnvVaultKeyName is the environment variable used to specify
|
||||
// the vault named key-ring. In the S3 context it's referred as
|
||||
// customer master key ID (CMK-ID).
|
||||
EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
|
||||
|
||||
// EnvVaultCAPath is the environment variable used to specify the
|
||||
// path to a directory of PEM-encoded CA cert files. These CA cert
|
||||
// files are used to authenticate Minio to Vault over mTLS.
|
||||
EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
|
||||
|
||||
// EnvVaultNamespace is the environment variable used to specify
|
||||
// vault namespace. The vault namespace is used if the enterprise
|
||||
// version of Hashicorp Vault is used.
|
||||
EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
|
||||
)
|
||||
|
||||
// Environment provides functions for accessing environment
|
||||
// variables.
|
||||
var Environment = environment{}
|
||||
|
||||
type environment struct{}
|
||||
|
||||
// Get retrieves the value of the environment variable named
|
||||
// by the key. If the variable is present in the environment the
|
||||
// value (which may be empty) is returned. Otherwise it returns
|
||||
// the specified default value.
|
||||
func (environment) Get(key, defaultValue string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
// Lookup retrieves the value of the environment variable named
|
||||
// by the key. If the variable is present in the environment the
|
||||
// value (which may be empty) is returned and the boolean is true.
|
||||
// Otherwise the returned value will be empty and the boolean will
|
||||
// be false.
|
||||
func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) }
|
||||
|
||||
// LookupKMSConfig extracts the KMS configuration provided by environment
|
||||
// variables and merge them with the provided KMS configuration. The
|
||||
// merging follows the following rules:
|
||||
//
|
||||
// 1. A valid value provided as environment variable is higher prioritized
|
||||
// than the provided configuration and overwrites the value from the
|
||||
// configuration file.
|
||||
//
|
||||
// 2. A value specified as environment variable never changes the configuration
|
||||
// file. So it is never made a persistent setting.
|
||||
//
|
||||
// It sets the global KMS configuration according to the merged configuration
|
||||
// on success.
|
||||
func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) {
|
||||
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
|
||||
config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
|
||||
config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
|
||||
config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
|
||||
config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
|
||||
config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
|
||||
config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
|
||||
config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
|
||||
keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
|
||||
config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
|
||||
}
|
||||
if err = config.Vault.Verify(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Lookup KMS master keys - only available through ENV.
|
||||
if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
|
||||
if !config.Vault.IsEmpty() { // Vault and KMS master key provided
|
||||
return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
|
||||
}
|
||||
globalKMSKeyID, GlobalKMS, err = parseKMSMasterKey(masterKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !config.Vault.IsEmpty() {
|
||||
GlobalKMS, err = crypto.NewVault(config.Vault)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalKMSKeyID = config.Vault.Key.Name
|
||||
}
|
||||
|
||||
autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalAutoEncryption = bool(autoEncryption)
|
||||
if globalAutoEncryption && GlobalKMS == nil { // auto-encryption enabled but no KMS
|
||||
return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseKMSMasterKey parses the value of the environment variable
|
||||
// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
|
||||
func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) {
|
||||
values := strings.SplitN(envArg, ":", 2)
|
||||
if len(values) != 2 {
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
|
||||
}
|
||||
var (
|
||||
keyID = values[0]
|
||||
hexKey = values[1]
|
||||
)
|
||||
if len(hexKey) != 64 { // 2 hex bytes = 1 byte
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
|
||||
}
|
||||
var masterKey [32]byte
|
||||
if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
|
||||
}
|
||||
return keyID, crypto.NewKMS(masterKey), nil
|
||||
}
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"github.com/klauspost/reedsolomon"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -82,7 +81,9 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
|
||||
if write < int64(len(block)) {
|
||||
n, err := io.Copy(dst, bytes.NewReader(block[:write]))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
if err != io.ErrClosedPipe {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
totalWritten += n
|
||||
@@ -92,7 +93,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
|
||||
n, err := io.Copy(dst, bytes.NewReader(block))
|
||||
if err != nil {
|
||||
// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
|
||||
if !strings.Contains(err.Error(), "read/write on closed pipe") {
|
||||
if err != io.ErrClosedPipe {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return 0, err
|
||||
|
||||
+34
-20
@@ -273,7 +273,7 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
|
||||
rlk.Close()
|
||||
return nil, err
|
||||
}
|
||||
logger.SetDeploymentID(id)
|
||||
globalDeploymentID = id
|
||||
return rlk, nil
|
||||
}
|
||||
}
|
||||
@@ -333,29 +333,43 @@ func formatFSFixDeploymentID(fsFormatPath string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
formatStartTime := time.Now().Round(time.Second)
|
||||
getElapsedTime := func() string {
|
||||
return time.Now().Round(time.Second).Sub(formatStartTime).String()
|
||||
}
|
||||
|
||||
doneCh := make(chan struct{})
|
||||
defer close(doneCh)
|
||||
|
||||
retryTimerCh := newRetryTimerSimple(doneCh)
|
||||
for {
|
||||
wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
|
||||
if err == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer wlk.Close()
|
||||
select {
|
||||
case <-retryTimerCh:
|
||||
|
||||
err = jsonLoad(wlk, format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
|
||||
if err == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again
|
||||
|
||||
// Check if it needs to be updated
|
||||
if format.ID != "" {
|
||||
return nil
|
||||
logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer wlk.Close()
|
||||
|
||||
err = jsonLoad(wlk, format)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Check if it needs to be updated
|
||||
if format.ID != "" {
|
||||
return nil
|
||||
}
|
||||
format.ID = mustGetUUID()
|
||||
return jsonSave(wlk, format)
|
||||
}
|
||||
format.ID = mustGetUUID()
|
||||
return jsonSave(wlk, format)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
+18
-3
@@ -379,6 +379,23 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
|
||||
return disk.RenameFile(minioMetaBucket, formatConfigFileTmp, minioMetaBucket, formatConfigFile)
|
||||
}
|
||||
|
||||
var ignoredHiddenDirectories = []string{
|
||||
minioMetaBucket,
|
||||
".snapshot",
|
||||
"lost+found",
|
||||
"$RECYCLE.BIN",
|
||||
"System Volume Information",
|
||||
}
|
||||
|
||||
func isIgnoreHiddenDirectories(dir string) bool {
|
||||
for _, ignDir := range ignoredHiddenDirectories {
|
||||
if dir == ignDir {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// loadFormatXL - loads format.json from disk.
|
||||
func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
|
||||
buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
|
||||
@@ -391,9 +408,7 @@ func loadFormatXL(disk StorageAPI) (format *formatXLV3, err error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(vols) > 1 || (len(vols) == 1 &&
|
||||
vols[0].Name != minioMetaBucket &&
|
||||
vols[0].Name != "lost+found") {
|
||||
if len(vols) > 1 || (len(vols) == 1 && !isIgnoreHiddenDirectories(vols[0].Name)) {
|
||||
// 'format.json' not found, but we
|
||||
// found user data.
|
||||
return nil, errCorruptedFormat
|
||||
|
||||
@@ -113,7 +113,7 @@ type fsMetaV1 struct {
|
||||
// Metadata map for current object.
|
||||
Meta map[string]string `json:"meta,omitempty"`
|
||||
// parts info for current object - used in encryption.
|
||||
Parts []objectPartInfo `json:"parts,omitempty"`
|
||||
Parts []ObjectPartInfo `json:"parts,omitempty"`
|
||||
}
|
||||
|
||||
// IsValid - tells if the format is sane by validating the version
|
||||
@@ -207,9 +207,9 @@ func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
|
||||
return metaMap
|
||||
}
|
||||
|
||||
func parseFSPartsArray(fsMetaBuf []byte) []objectPartInfo {
|
||||
func parseFSPartsArray(fsMetaBuf []byte) []ObjectPartInfo {
|
||||
// Get xlMetaV1.Parts array
|
||||
var partsArray []objectPartInfo
|
||||
var partsArray []ObjectPartInfo
|
||||
|
||||
partsArrayResult := gjson.GetBytes(fsMetaBuf, "parts")
|
||||
partsArrayResult.ForEach(func(key, part gjson.Result) bool {
|
||||
@@ -219,7 +219,7 @@ func parseFSPartsArray(fsMetaBuf []byte) []objectPartInfo {
|
||||
etag := gjson.Get(partJSON, "etag").String()
|
||||
size := gjson.Get(partJSON, "size").Int()
|
||||
actualSize := gjson.Get(partJSON, "actualSize").Int()
|
||||
partsArray = append(partsArray, objectPartInfo{
|
||||
partsArray = append(partsArray, ObjectPartInfo{
|
||||
Number: int(number),
|
||||
Name: name,
|
||||
ETag: etag,
|
||||
|
||||
@@ -54,7 +54,7 @@ func TestReadFSMetadata(t *testing.T) {
|
||||
if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
|
||||
if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ func TestWriteFSMetadata(t *testing.T) {
|
||||
if err := obj.MakeBucketWithLocation(context.Background(), bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
|
||||
if _, err := obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{}); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
|
||||
+23
-10
@@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
@@ -31,8 +30,6 @@ import (
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
mioutil "github.com/minio/minio/pkg/ioutil"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// Returns EXPORT/.minio.sys/multipart/SHA256/UPLOADID
|
||||
@@ -259,7 +256,7 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
|
||||
return pi, toObjectErr(err)
|
||||
}
|
||||
|
||||
partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.Reader, dstOpts)
|
||||
partInfo, err := fs.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
|
||||
if err != nil {
|
||||
return pi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
@@ -271,7 +268,8 @@ func (fs *FSObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
|
||||
// an ongoing multipart transaction. Internally incoming data is
|
||||
// written to '.minio.sys/tmp' location and safely renamed to
|
||||
// '.minio.sys/multipart' for reach parts.
|
||||
func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, e error) {
|
||||
func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, e error) {
|
||||
data := r.Reader
|
||||
if err := checkPutObjectPartArgs(ctx, bucket, object, fs); err != nil {
|
||||
return pi, toObjectErr(err, bucket)
|
||||
}
|
||||
@@ -321,11 +319,12 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
// PutObjectPart succeeds then there would be nothing to
|
||||
// delete in which case we just ignore the error.
|
||||
defer fsRemoveFile(ctx, tmpPartPath)
|
||||
etag := r.MD5CurrentHexString()
|
||||
|
||||
etag := hex.EncodeToString(data.MD5Current())
|
||||
if etag == "" {
|
||||
etag = GenETag()
|
||||
}
|
||||
|
||||
partPath := pathJoin(uploadIDDir, fs.encodePartFile(partID, etag, data.ActualSize()))
|
||||
|
||||
if err = fsRenameFile(ctx, tmpPartPath, partPath); err != nil {
|
||||
@@ -354,7 +353,7 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
// Implements S3 compatible ListObjectParts API. The resulting
|
||||
// ListPartsInfo structure is unmarshalled directly into XML and
|
||||
// replied back to the client.
|
||||
func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (result ListPartsInfo, e error) {
|
||||
func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
|
||||
if err := checkListPartsArgs(ctx, bucket, object, fs); err != nil {
|
||||
return result, toObjectErr(err)
|
||||
}
|
||||
@@ -478,7 +477,7 @@ func (fs *FSObjects) ListObjectParts(ctx context.Context, bucket, object, upload
|
||||
// md5sums of all the parts.
|
||||
//
|
||||
// Implements S3 compatible Complete multipart API.
|
||||
func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart) (oi ObjectInfo, e error) {
|
||||
func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, e error) {
|
||||
|
||||
var actualSize int64
|
||||
|
||||
@@ -516,7 +515,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
fsMeta := fsMetaV1{}
|
||||
|
||||
// Allocate parts similar to incoming slice.
|
||||
fsMeta.Parts = make([]objectPartInfo, len(parts))
|
||||
fsMeta.Parts = make([]ObjectPartInfo, len(parts))
|
||||
|
||||
entries, err := readDir(uploadIDDir)
|
||||
if err != nil {
|
||||
@@ -561,7 +560,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
partSize = actualSize
|
||||
}
|
||||
|
||||
fsMeta.Parts[i] = objectPartInfo{
|
||||
fsMeta.Parts[i] = ObjectPartInfo{
|
||||
Number: part.PartNumber,
|
||||
ETag: part.ETag,
|
||||
Size: fi.Size(),
|
||||
@@ -683,6 +682,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
fsRemoveAll(ctx, uploadIDDir)
|
||||
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
||||
fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
|
||||
fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
@@ -713,6 +714,11 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
|
||||
}
|
||||
|
||||
fs.appendFileMapMu.Lock()
|
||||
// Remove file in tmp folder
|
||||
file := fs.appendFileMap[uploadID]
|
||||
if file != nil {
|
||||
fsRemoveFile(ctx, file.filePath)
|
||||
}
|
||||
delete(fs.appendFileMap, uploadID)
|
||||
fs.appendFileMapMu.Unlock()
|
||||
|
||||
@@ -725,10 +731,14 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
|
||||
}
|
||||
return toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
// Ignore the error returned as Windows fails to remove directory if a file in it
|
||||
// is Open()ed by the backgroundAppend()
|
||||
fsRemoveAll(ctx, uploadIDDir)
|
||||
|
||||
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
||||
fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -761,6 +771,9 @@ func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupIn
|
||||
}
|
||||
if now.Sub(fi.ModTime()) > expiry {
|
||||
fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
|
||||
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
||||
fsRemoveDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,7 +36,7 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
|
||||
|
||||
// Close the go-routine, we are going to
|
||||
// manually start it and test in this test case.
|
||||
globalServiceDoneCh <- struct{}{}
|
||||
GlobalServiceDoneCh <- struct{}{}
|
||||
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
@@ -47,14 +47,14 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, globalServiceDoneCh)
|
||||
go fs.cleanupStaleMultipartUploads(context.Background(), 20*time.Millisecond, 0, GlobalServiceDoneCh)
|
||||
|
||||
// Wait for 40ms such that - we have given enough time for
|
||||
// cleanup routine to kick in.
|
||||
time.Sleep(40 * time.Millisecond)
|
||||
|
||||
// Close the routine we do not need it anymore.
|
||||
globalServiceDoneCh <- struct{}{}
|
||||
GlobalServiceDoneCh <- struct{}{}
|
||||
|
||||
// Check if upload id was already purged.
|
||||
if err = obj.AbortMultipartUpload(context.Background(), bucketName, objectName, uploadID); err != nil {
|
||||
@@ -114,7 +114,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
|
||||
sha256sum := ""
|
||||
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
|
||||
_, err = fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{})
|
||||
if !isSameType(err, BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
@@ -145,7 +145,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
|
||||
|
||||
parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts); err != nil {
|
||||
if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
|
||||
if !isSameType(err, BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
@@ -175,13 +175,13 @@ func TestCompleteMultipartUpload(t *testing.T) {
|
||||
|
||||
md5Hex := getMD5Hash(data)
|
||||
|
||||
if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
|
||||
if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
|
||||
|
||||
if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts); err != nil {
|
||||
if _, err := fs.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
}
|
||||
@@ -209,7 +209,7 @@ func TestAbortMultipartUpload(t *testing.T) {
|
||||
|
||||
md5Hex := getMD5Hash(data)
|
||||
|
||||
if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
|
||||
if _, err := fs.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
|
||||
|
||||
+36
-33
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@@ -32,7 +31,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
@@ -154,10 +152,10 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
fs.fsFormatRlk = rlk
|
||||
|
||||
if !fs.diskMount {
|
||||
go fs.diskUsage(globalServiceDoneCh)
|
||||
go fs.diskUsage(GlobalServiceDoneCh)
|
||||
}
|
||||
|
||||
go fs.cleanupStaleMultipartUploads(ctx, globalMultipartCleanupInterval, globalMultipartExpiry, globalServiceDoneCh)
|
||||
go fs.cleanupStaleMultipartUploads(ctx, GlobalMultipartCleanupInterval, GlobalMultipartExpiry, GlobalServiceDoneCh)
|
||||
|
||||
// Return successfully initialized object layer.
|
||||
return fs, nil
|
||||
@@ -189,14 +187,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
case <-doneCh:
|
||||
return errWalkAbort
|
||||
default:
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
}
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
|
||||
@@ -228,14 +221,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
}
|
||||
}
|
||||
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
}
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
usage = usage + uint64(fi.Size())
|
||||
@@ -461,8 +449,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
|
||||
// Return the new object info.
|
||||
return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
|
||||
}
|
||||
|
||||
objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.Reader, srcInfo.UserDefined)
|
||||
objInfo, err := fs.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, dstOpts)
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
@@ -719,8 +706,12 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
|
||||
// Read from fs metadata only if it exists.
|
||||
_, rerr := fsMeta.ReadFrom(ctx, rlk.LockedFile)
|
||||
fs.rwPool.Close(fsMetaPath)
|
||||
if rerr != nil && rerr != io.EOF {
|
||||
return oi, rerr
|
||||
if rerr != nil {
|
||||
if rerr != io.EOF {
|
||||
return oi, rerr
|
||||
}
|
||||
// Set Default ETag, if fs.json is empty
|
||||
fsMeta = fs.defaultFsJSON(object)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -813,8 +804,8 @@ func (fs *FSObjects) parentDirIsObject(ctx context.Context, bucket, parent strin
|
||||
// until EOF, writes data directly to configured filesystem path.
|
||||
// Additionally writes `fs.json` which carries the necessary metadata
|
||||
// for future object operations.
|
||||
func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
|
||||
if err := checkPutObjectArgs(ctx, bucket, object, fs, data.Size()); err != nil {
|
||||
func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
|
||||
if err := checkPutObjectArgs(ctx, bucket, object, fs, r.Size()); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
// Lock the object.
|
||||
@@ -824,11 +815,13 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
|
||||
return objInfo, err
|
||||
}
|
||||
defer objectLock.Unlock()
|
||||
return fs.putObject(ctx, bucket, object, data, metadata)
|
||||
return fs.putObject(ctx, bucket, object, r, metadata, opts)
|
||||
}
|
||||
|
||||
// putObject - wrapper for PutObject
|
||||
func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
|
||||
func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, metadata map[string]string, opts ObjectOptions) (objInfo ObjectInfo, retErr error) {
|
||||
data := r.Reader
|
||||
|
||||
// No metadata is set, allocate a new one.
|
||||
meta := make(map[string]string)
|
||||
for k, v := range metadata {
|
||||
@@ -919,8 +912,7 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
|
||||
fsRemoveFile(ctx, fsTmpObjPath)
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
fsMeta.Meta["etag"] = hex.EncodeToString(data.MD5Current())
|
||||
fsMeta.Meta["etag"] = r.MD5CurrentHexString()
|
||||
|
||||
// Should return IncompleteBody{} error when reader has fewer
|
||||
// bytes than specified in request header.
|
||||
@@ -1018,7 +1010,7 @@ func (fs *FSObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
|
||||
listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool) {
|
||||
var err error
|
||||
entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
|
||||
if err != nil {
|
||||
if err != nil && err != errFileNotFound {
|
||||
logger.LogIf(context.Background(), err)
|
||||
return
|
||||
}
|
||||
@@ -1220,9 +1212,8 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
|
||||
fs.listPool.Set(params, walkResultCh, endWalkCh)
|
||||
}
|
||||
|
||||
result := ListObjectsInfo{IsTruncated: !eof}
|
||||
result := ListObjectsInfo{}
|
||||
for _, objInfo := range objInfos {
|
||||
result.NextMarker = objInfo.Name
|
||||
if objInfo.IsDir && delimiter == slashSeparator {
|
||||
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
||||
continue
|
||||
@@ -1230,6 +1221,13 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
|
||||
result.Objects = append(result.Objects, objInfo)
|
||||
}
|
||||
|
||||
if !eof {
|
||||
result.IsTruncated = true
|
||||
if len(objInfos) > 0 {
|
||||
result.NextMarker = objInfos[len(objInfos)-1].Name
|
||||
}
|
||||
}
|
||||
|
||||
// Success.
|
||||
return result, nil
|
||||
}
|
||||
@@ -1274,7 +1272,7 @@ func (fs *FSObjects) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
|
||||
|
||||
// SetBucketPolicy sets policy on bucket
|
||||
func (fs *FSObjects) SetBucketPolicy(ctx context.Context, bucket string, policy *policy.Policy) error {
|
||||
return savePolicyConfig(fs, bucket, policy)
|
||||
return savePolicyConfig(ctx, fs, bucket, policy)
|
||||
}
|
||||
|
||||
// GetBucketPolicy will get policy on bucket
|
||||
@@ -1314,7 +1312,12 @@ func (fs *FSObjects) IsNotificationSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (fs *FSObjects) IsListenBucketSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
|
||||
func (fs *FSObjects) IsEncryptionSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
+9
-9
@@ -40,7 +40,7 @@ func TestFSParentDirIsObject(t *testing.T) {
|
||||
}
|
||||
objectContent := "12345"
|
||||
objInfo, err := obj.PutObject(context.Background(), bucketName, objectName,
|
||||
mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
|
||||
mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -124,7 +124,7 @@ func TestFSShutdown(t *testing.T) {
|
||||
|
||||
objectContent := "12345"
|
||||
obj.MakeBucketWithLocation(context.Background(), bucketName, "")
|
||||
obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
|
||||
obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte(objectContent)), int64(len(objectContent)), "", ""), nil, ObjectOptions{})
|
||||
return fs, disk
|
||||
}
|
||||
|
||||
@@ -203,7 +203,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
}
|
||||
|
||||
// With a regular object.
|
||||
_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
_, err := obj.PutObject(context.Background(), bucketName+"non-existent", objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
@@ -212,7 +212,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
}
|
||||
|
||||
// With a directory object.
|
||||
_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
|
||||
_, err = obj.PutObject(context.Background(), bucketName+"non-existent", objectName+"/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
@@ -220,11 +220,11 @@ func TestFSPutObject(t *testing.T) {
|
||||
t.Fatalf("Expected error type BucketNotFound, got %#v", err)
|
||||
}
|
||||
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backend corruption occurred")
|
||||
}
|
||||
@@ -239,7 +239,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetHashReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
|
||||
_, err = obj.PutObject(context.Background(), bucketName, objectName+"/1/", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), 0, "", ""), nil, ObjectOptions{})
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backned corruption occurred")
|
||||
}
|
||||
@@ -267,7 +267,7 @@ func TestFSDeleteObject(t *testing.T) {
|
||||
objectName := "object"
|
||||
|
||||
obj.MakeBucketWithLocation(context.Background(), bucketName, "")
|
||||
obj.PutObject(context.Background(), bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
obj.PutObject(context.Background(), bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil, ObjectOptions{})
|
||||
|
||||
// Test with invalid bucket name
|
||||
if err := fs.DeleteObject(context.Background(), "fo", objectName); !isSameType(err, BucketNameInvalid{}) {
|
||||
@@ -354,7 +354,7 @@ func TestFSListBuckets(t *testing.T) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
globalServiceDoneCh <- struct{}{}
|
||||
GlobalServiceDoneCh <- struct{}{}
|
||||
|
||||
// Create a bucket with invalid name
|
||||
if err := os.MkdirAll(pathJoin(fs.fsPath, "vo^"), 0777); err != nil {
|
||||
|
||||
+61
-1
@@ -18,7 +18,10 @@ package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
@@ -30,8 +33,27 @@ var (
|
||||
|
||||
// MustGetUUID function alias.
|
||||
MustGetUUID = mustGetUUID
|
||||
|
||||
// IsMinAllowedPartSize function alias.
|
||||
IsMinAllowedPartSize = isMinAllowedPartSize
|
||||
|
||||
// GetCompleteMultipartMD5 functon alias.
|
||||
GetCompleteMultipartMD5 = getCompleteMultipartMD5
|
||||
|
||||
// Contains function alias.
|
||||
Contains = contains
|
||||
|
||||
// ExtractETag provides extractETag function alias.
|
||||
ExtractETag = extractETag
|
||||
// CleanMetadataKeys provides cleanMetadataKeys function alias.
|
||||
CleanMetadataKeys = cleanMetadataKeys
|
||||
)
|
||||
|
||||
// StatInfo - alias for statInfo
|
||||
type StatInfo struct {
|
||||
statInfo
|
||||
}
|
||||
|
||||
// AnonErrToObjectErr - converts standard http codes into meaningful object layer errors.
|
||||
func AnonErrToObjectErr(statusCode int, params ...string) error {
|
||||
bucket := ""
|
||||
@@ -224,7 +246,16 @@ func FromMinioClientListBucketResultToV2Info(bucket string, result minio.ListBuc
|
||||
}
|
||||
}
|
||||
|
||||
// ToMinioClientMetadata converts metadata to map[string][]string
|
||||
// ToMinioClientObjectInfoMetadata convertes metadata to map[string][]string
|
||||
func ToMinioClientObjectInfoMetadata(metadata map[string]string) map[string][]string {
|
||||
mm := make(map[string][]string, len(metadata))
|
||||
for k, v := range metadata {
|
||||
mm[http.CanonicalHeaderKey(k)] = []string{v}
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// ToMinioClientMetadata converts metadata to map[string]string
|
||||
func ToMinioClientMetadata(metadata map[string]string) map[string]string {
|
||||
mm := make(map[string]string)
|
||||
for k, v := range metadata {
|
||||
@@ -285,6 +316,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
|
||||
err = BucketPolicyNotFound{}
|
||||
case "InvalidBucketName":
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
case "InvalidPart":
|
||||
err = InvalidPart{}
|
||||
case "NoSuchBucket":
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
case "NoSuchKey":
|
||||
@@ -310,3 +343,30 @@ func ErrorRespToObjectError(err error, params ...string) error {
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// parse gateway sse env variable
|
||||
func parseGatewaySSE(s string) (gatewaySSE, error) {
|
||||
l := strings.Split(s, ";")
|
||||
var gwSlice = make([]string, 0)
|
||||
for _, val := range l {
|
||||
v := strings.ToUpper(val)
|
||||
if v == gatewaySSES3 || v == gatewaySSEC {
|
||||
gwSlice = append(gwSlice, v)
|
||||
continue
|
||||
}
|
||||
return nil, uiErrInvalidGWSSEValue(nil).Msg("gateway SSE cannot be (%s) ", v)
|
||||
}
|
||||
return gatewaySSE(gwSlice), nil
|
||||
}
|
||||
|
||||
// handle gateway env vars
|
||||
func handleGatewayEnvVars() {
|
||||
gwsseVal, ok := os.LookupEnv("MINIO_GATEWAY_SSE")
|
||||
if ok {
|
||||
var err error
|
||||
GlobalGatewaySSE, err = parseGatewaySSE(gwsseVal)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to parse MINIO_GATEWAY_SSE value (`%s`)", gwsseVal)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests cache exclude parsing.
|
||||
func TestParseGatewaySSE(t *testing.T) {
|
||||
testCases := []struct {
|
||||
gwSSEStr string
|
||||
expected gatewaySSE
|
||||
success bool
|
||||
}{
|
||||
// valid input
|
||||
{"c;S3", []string{"C", "S3"}, true},
|
||||
{"S3", []string{"S3"}, true},
|
||||
{"c,S3", []string{}, false},
|
||||
{"c;S3;KMS", []string{}, false},
|
||||
{"C;s3", []string{"C", "S3"}, true},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
gwSSE, err := parseGatewaySSE(testCase.gwSSEStr)
|
||||
if err != nil && testCase.success {
|
||||
t.Errorf("Test %d: Expected success but failed instead %s", i+1, err)
|
||||
}
|
||||
if err == nil && !testCase.success {
|
||||
t.Errorf("Test %d: Expected failure but passed instead", i+1)
|
||||
}
|
||||
if err == nil {
|
||||
if !reflect.DeepEqual(gwSSE, testCase.expected) {
|
||||
t.Errorf("Test %d: Expected %v, got %v", i+1, testCase.expected, gwSSE)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
type gatewaySSE []string
|
||||
|
||||
const (
|
||||
// GatewaySSES3 is set when SSE-S3 encryption needed on both gateway and backend
|
||||
gatewaySSES3 = "S3"
|
||||
// GatewaySSEC is set when SSE-C encryption needed on both gateway and backend
|
||||
gatewaySSEC = "C"
|
||||
)
|
||||
|
||||
func (sse gatewaySSE) SSES3() bool {
|
||||
for _, v := range sse {
|
||||
if v == gatewaySSES3 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (sse gatewaySSE) SSEC() bool {
|
||||
for _, v := range sse {
|
||||
if v == gatewaySSEC {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (sse gatewaySSE) IsSet() bool {
|
||||
return sse.SSES3() || sse.SSEC()
|
||||
}
|
||||
+117
-71
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -41,7 +40,7 @@ func init() {
|
||||
var (
|
||||
gatewayCmd = cli.Command{
|
||||
Name: "gateway",
|
||||
Usage: "Start object storage gateway.",
|
||||
Usage: "start object storage gateway",
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
@@ -112,91 +111,54 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if jsonFlag {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if quietFlag {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Fetch address option
|
||||
gatewayAddr := ctx.GlobalString("address")
|
||||
if gatewayAddr == ":"+globalMinioPort {
|
||||
gatewayAddr = ctx.String("address")
|
||||
}
|
||||
|
||||
// Handle common command args.
|
||||
handleCommonCmdArgs(ctx)
|
||||
|
||||
// Handle common env vars.
|
||||
handleCommonEnvVars()
|
||||
|
||||
// Get port to listen on from gateway address
|
||||
_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
|
||||
if pErr != nil {
|
||||
logger.FatalIf(pErr, "Unable to start gateway")
|
||||
}
|
||||
globalMinioHost, globalMinioPort = mustSplitHostPort(globalCLIContext.Addr)
|
||||
|
||||
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
|
||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error situation we check for port availability.
|
||||
logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
|
||||
logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the gateway")
|
||||
|
||||
// Check and load TLS certificates.
|
||||
var err error
|
||||
globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
|
||||
logger.FatalIf(err, "Invalid TLS certificate file")
|
||||
|
||||
// Check and load Root CAs.
|
||||
globalRootCAs, err = getRootCAs(globalCertsCADir.Get())
|
||||
logger.FatalIf(err, "Failed to read root CAs (%v)", err)
|
||||
|
||||
// Handle common env vars.
|
||||
handleCommonEnvVars()
|
||||
|
||||
// Handle gateway specific env
|
||||
handleGatewayEnvVars()
|
||||
|
||||
// Validate if we have access, secret set through environment.
|
||||
if !globalIsEnvCreds {
|
||||
logger.Fatal(uiErrEnvCredentialsMissingGateway(nil), "Unable to start gateway")
|
||||
}
|
||||
|
||||
// Create certs path.
|
||||
logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
|
||||
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// Override any values from ENVs.
|
||||
srvCfg.loadFromEnvs()
|
||||
|
||||
// Load values to cached global values.
|
||||
srvCfg.loadToCachedConfigs()
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
|
||||
var cacheConfig = globalServerConfig.GetCacheConfig()
|
||||
if len(cacheConfig.Drives) > 0 {
|
||||
var err error
|
||||
// initialize the new disk cache objects.
|
||||
globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
|
||||
logger.FatalIf(err, "Unable to initialize disk caching")
|
||||
}
|
||||
|
||||
// Check and load SSL certificates.
|
||||
var err error
|
||||
globalPublicCerts, globalRootCAs, globalTLSCerts, globalIsSSL, err = getSSLConfig()
|
||||
logger.FatalIf(err, "Invalid SSL certificate file")
|
||||
|
||||
// Set system resources to maximum.
|
||||
logger.LogIf(context.Background(), setMaxResources())
|
||||
|
||||
initNSLock(false) // Enable local namespace lock.
|
||||
|
||||
// Create new notification system.
|
||||
globalNotificationSys = NewNotificationSys(globalServerConfig, EndpointList{})
|
||||
|
||||
// Create new policy system.
|
||||
globalPolicySys = NewPolicySys()
|
||||
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
|
||||
if globalEtcdClient != nil {
|
||||
// Enable STS router if etcd is enabled.
|
||||
registerSTSRouter(router)
|
||||
}
|
||||
|
||||
// Enable IAM admin APIs if etcd is enabled, if not just enable basic
|
||||
// operations such as profiling, server info etc.
|
||||
registerAdminRouter(router, globalEtcdClient != nil)
|
||||
|
||||
// Add healthcheck router
|
||||
registerHealthCheckRouter(router)
|
||||
|
||||
@@ -208,15 +170,27 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
logger.FatalIf(registerWebRouter(router), "Unable to configure web browser")
|
||||
}
|
||||
|
||||
// Currently only NAS and S3 gateway support encryption headers.
|
||||
encryptionEnabled := gatewayName == "s3" || gatewayName == "nas"
|
||||
|
||||
// Add API router.
|
||||
registerAPIRouter(router)
|
||||
registerAPIRouter(router, encryptionEnabled)
|
||||
|
||||
// Dummy endpoint representing gateway instance.
|
||||
globalEndpoints = []Endpoint{{
|
||||
URL: &url.URL{Path: "/minio/gateway"},
|
||||
IsLocal: true,
|
||||
}}
|
||||
|
||||
// Initialize Admin Peers.
|
||||
initGlobalAdminPeers(globalEndpoints)
|
||||
|
||||
var getCert certs.GetCertificateFunc
|
||||
if globalTLSCerts != nil {
|
||||
getCert = globalTLSCerts.GetCertificate
|
||||
}
|
||||
|
||||
globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
|
||||
globalHTTPServer = xhttp.NewServer([]string{globalCLIContext.Addr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
|
||||
globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
|
||||
globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
|
||||
go func() {
|
||||
@@ -225,6 +199,22 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
// !!! Do not move this block !!!
|
||||
// For all gateways, the config needs to be loaded from env
|
||||
// prior to initializing the gateway layer
|
||||
{
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// Override any values from ENVs.
|
||||
srvCfg.loadFromEnvs()
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
}
|
||||
|
||||
newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
|
||||
if err != nil {
|
||||
// Stop watching for any certificate changes.
|
||||
@@ -234,15 +224,71 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
logger.FatalIf(err, "Unable to initialize gateway backend")
|
||||
}
|
||||
|
||||
if globalEtcdClient != nil && gatewayName == "nas" {
|
||||
// Create a new config system.
|
||||
globalConfigSys = NewConfigSys()
|
||||
|
||||
// Load globalServerConfig from etcd
|
||||
_ = globalConfigSys.Init(newObject)
|
||||
}
|
||||
|
||||
// Load logger subsystem
|
||||
loadLoggers()
|
||||
|
||||
// This is only to uniquely identify each gateway deployments.
|
||||
globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")
|
||||
|
||||
var cacheConfig = globalServerConfig.GetCacheConfig()
|
||||
if len(cacheConfig.Drives) > 0 {
|
||||
var err error
|
||||
// initialize the new disk cache objects.
|
||||
globalCacheObjectAPI, err = newServerCacheObjects(cacheConfig)
|
||||
logger.FatalIf(err, "Unable to initialize disk caching")
|
||||
}
|
||||
|
||||
// Re-enable logging
|
||||
logger.Disable = false
|
||||
|
||||
// Create new IAM system.
|
||||
globalIAMSys = NewIAMSys()
|
||||
if globalEtcdClient != nil {
|
||||
// Initialize IAM sys.
|
||||
_ = globalIAMSys.Init(newObject)
|
||||
}
|
||||
|
||||
// Create new policy system.
|
||||
globalPolicySys = NewPolicySys()
|
||||
|
||||
// Initialize policy system.
|
||||
go globalPolicySys.Init(newObject)
|
||||
|
||||
// Create new notification system.
|
||||
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
|
||||
if globalEtcdClient != nil && newObject.IsNotificationSupported() {
|
||||
_ = globalNotificationSys.Init(newObject)
|
||||
}
|
||||
|
||||
// Encryption support checks in gateway mode.
|
||||
{
|
||||
|
||||
if (globalAutoEncryption || GlobalKMS != nil) && !newObject.IsEncryptionSupported() {
|
||||
logger.Fatal(errInvalidArgument,
|
||||
"Encryption support is requested but (%s) gateway does not support encryption", gw.Name())
|
||||
}
|
||||
|
||||
if GlobalGatewaySSE.IsSet() && GlobalKMS == nil {
|
||||
logger.Fatal(uiErrInvalidGWSSEEnvValue(nil).Msg("MINIO_GATEWAY_SSE set but KMS is not configured"),
|
||||
"Unable to start gateway with SSE")
|
||||
}
|
||||
}
|
||||
|
||||
// Once endpoints are finalized, initialize the new object api.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = newObject
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
if !quietFlag {
|
||||
if !globalCLIContext.Quiet {
|
||||
mode := globalMinioModeGatewayPrefix + gatewayName
|
||||
// Check update mode.
|
||||
checkUpdate(mode)
|
||||
@@ -253,11 +299,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
}
|
||||
|
||||
// Print gateway startup message.
|
||||
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
|
||||
printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
|
||||
}
|
||||
|
||||
// Reenable logging
|
||||
logger.Disable = false
|
||||
// Set uptime time after object layer has initialized.
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
handleSignals()
|
||||
}
|
||||
|
||||
@@ -55,10 +55,14 @@ func printGatewayCommonMsg(apiEndpoints []string) {
|
||||
cred := globalServerConfig.GetCredential()
|
||||
|
||||
apiEndpointStr := strings.Join(apiEndpoints, " ")
|
||||
|
||||
// Colorize the message and print.
|
||||
logger.StartupMessage(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
if isTerminal() && !globalCLIContext.Anonymous {
|
||||
logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
}
|
||||
printEventNotifiers()
|
||||
|
||||
if globalIsBrowserEnabled {
|
||||
logger.StartupMessage(colorBlue("\nBrowser Access:"))
|
||||
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
)
|
||||
@@ -30,42 +29,38 @@ type GatewayUnsupported struct{}
|
||||
|
||||
// ListMultipartUploads lists all multipart uploads.
|
||||
func (a GatewayUnsupported) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return lmi, NotImplemented{}
|
||||
}
|
||||
|
||||
// NewMultipartUpload upload object in multiple parts
|
||||
func (a GatewayUnsupported) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, opts ObjectOptions) (uploadID string, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return "", NotImplemented{}
|
||||
}
|
||||
|
||||
// CopyObjectPart copy part of object to uploadID for another object
|
||||
func (a GatewayUnsupported) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return pi, NotImplemented{}
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts ObjectOptions) (pi PartInfo, err error) {
|
||||
func (a GatewayUnsupported) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return pi, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
|
||||
func (a GatewayUnsupported) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (lpi ListPartsInfo, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return lpi, NotImplemented{}
|
||||
}
|
||||
|
||||
// AbortMultipartUpload aborts a ongoing multipart upload
|
||||
func (a GatewayUnsupported) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return NotImplemented{}
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) {
|
||||
func (a GatewayUnsupported) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return oi, NotImplemented{}
|
||||
}
|
||||
@@ -78,13 +73,11 @@ func (a GatewayUnsupported) SetBucketPolicy(ctx context.Context, bucket string,
|
||||
|
||||
// GetBucketPolicy will get policy on bucket
|
||||
func (a GatewayUnsupported) GetBucketPolicy(ctx context.Context, bucket string) (bucketPolicy *policy.Policy, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return nil, NotImplemented{}
|
||||
}
|
||||
|
||||
// DeleteBucketPolicy deletes all policies on bucket
|
||||
func (a GatewayUnsupported) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return NotImplemented{}
|
||||
}
|
||||
|
||||
@@ -95,50 +88,42 @@ func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error
|
||||
|
||||
// HealFormat - Not implemented stub
|
||||
func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return madmin.HealResultItem{}, NotImplemented{}
|
||||
}
|
||||
|
||||
// HealBucket - Not implemented stub
|
||||
func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return nil, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not implemented stub
|
||||
func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return nil, NotImplemented{}
|
||||
}
|
||||
|
||||
// HealObject - Not implemented stub
|
||||
func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return h, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListObjectsV2 - Not implemented stub
|
||||
func (a GatewayUnsupported) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return result, NotImplemented{}
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not implemented stub
|
||||
func (a GatewayUnsupported) ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return loi, NotImplemented{}
|
||||
}
|
||||
|
||||
// CopyObject copies a blob from source container to destination container.
|
||||
func (a GatewayUnsupported) CopyObject(ctx context.Context, srcBucket string, srcObject string, destBucket string, destObject string,
|
||||
srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return objInfo, NotImplemented{}
|
||||
}
|
||||
|
||||
// RefreshBucketPolicy refreshes cache policy with what's on disk.
|
||||
func (a GatewayUnsupported) RefreshBucketPolicy(ctx context.Context, bucket string) error {
|
||||
logger.LogIf(ctx, NotImplemented{})
|
||||
return NotImplemented{}
|
||||
}
|
||||
|
||||
@@ -147,7 +132,12 @@ func (a GatewayUnsupported) IsNotificationSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (a GatewayUnsupported) IsListenBucketSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
|
||||
func (a GatewayUnsupported) IsEncryptionSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
@@ -32,12 +32,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
"github.com/Azure/go-autorest/autorest/azure"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
"github.com/minio/minio/pkg/policy/condition"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
@@ -93,7 +93,7 @@ EXAMPLES:
|
||||
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}} https://azure.example.com
|
||||
$ {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
|
||||
|
||||
3. Start minio gateway server for Azure Blob Storage backend with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
@@ -140,28 +140,43 @@ func (g *Azure) Name() string {
|
||||
return azureBackend
|
||||
}
|
||||
|
||||
// All known cloud environments of Azure
|
||||
var azureEnvs = []azure.Environment{
|
||||
azure.PublicCloud,
|
||||
azure.USGovernmentCloud,
|
||||
azure.ChinaCloud,
|
||||
azure.GermanCloud,
|
||||
}
|
||||
|
||||
// NewGatewayLayer initializes azure blob storage client and returns AzureObjects.
|
||||
func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error) {
|
||||
var err error
|
||||
var endpoint = storage.DefaultBaseURL
|
||||
// The default endpoint is the public cloud
|
||||
var endpoint = azure.PublicCloud.StorageEndpointSuffix
|
||||
var secure = true
|
||||
|
||||
// If user provided some parameters
|
||||
// Load the endpoint url if supplied by the user.
|
||||
if g.host != "" {
|
||||
endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Reformat the full account storage endpoint to the base format.
|
||||
// e.g. testazure.blob.core.windows.net => core.windows.net
|
||||
endpoint = strings.ToLower(endpoint)
|
||||
for _, env := range azureEnvs {
|
||||
if strings.Contains(endpoint, env.StorageEndpointSuffix) {
|
||||
endpoint = env.StorageEndpointSuffix
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if endpoint == fmt.Sprintf("%s.blob.%s", creds.AccessKey, storage.DefaultBaseURL) {
|
||||
// If, by mistake, user provides endpoint as accountname.blob.core.windows.net
|
||||
endpoint = storage.DefaultBaseURL
|
||||
}
|
||||
c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
|
||||
if err != nil {
|
||||
return &azureObjects{}, err
|
||||
}
|
||||
|
||||
c.AddToUserAgent(fmt.Sprintf("APN/1.0 Minio/1.0 Minio/%s", minio.Version))
|
||||
c.HTTPClient = &http.Client{Transport: minio.NewCustomHTTPTransport()}
|
||||
|
||||
@@ -672,7 +687,6 @@ func (a *azureObjects) GetObject(ctx context.Context, bucket, object string, sta
|
||||
}
|
||||
_, err = io.Copy(writer, rc)
|
||||
rc.Close()
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -722,7 +736,8 @@ func (a *azureObjects) GetObjectInfo(ctx context.Context, bucket, object string,
|
||||
|
||||
// PutObject - Create a new blob with the incoming data,
|
||||
// uses Azure equivalent CreateBlockBlobFromReader.
|
||||
func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
data := r.Reader
|
||||
if data.Size() < azureBlockSize/10 {
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(ctx, metadata)
|
||||
@@ -923,7 +938,8 @@ func (a *azureObjects) NewMultipartUpload(ctx context.Context, bucket, object st
|
||||
}
|
||||
|
||||
// PutObjectPart - Use Azure equivalent PutBlockWithLength.
|
||||
func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
|
||||
func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (info minio.PartInfo, err error) {
|
||||
data := r.Reader
|
||||
if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
|
||||
return info, err
|
||||
}
|
||||
@@ -965,7 +981,7 @@ func (a *azureObjects) PutObjectPart(ctx context.Context, bucket, object, upload
|
||||
}
|
||||
|
||||
// ListObjectParts - Use Azure equivalent GetBlockList.
|
||||
func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int) (result minio.ListPartsInfo, err error) {
|
||||
func (a *azureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (result minio.ListPartsInfo, err error) {
|
||||
if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -1062,7 +1078,7 @@ func (a *azureObjects) AbortMultipartUpload(ctx context.Context, bucket, object,
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - Use Azure equivalent PutBlockList.
|
||||
func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart) (objInfo minio.ObjectInfo, err error) {
|
||||
func (a *azureObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
metadataObject := getAzureMetadataObjectName(object, uploadID)
|
||||
if err = a.checkUploadIDExists(ctx, bucket, object, uploadID); err != nil {
|
||||
return objInfo, err
|
||||
|
||||
@@ -92,7 +92,7 @@ EXAMPLES:
|
||||
`
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: b2Backend,
|
||||
Usage: "Backblaze B2.",
|
||||
Usage: "Backblaze B2",
|
||||
Action: b2GatewayMain,
|
||||
CustomHelpTemplate: b2GatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -534,7 +534,9 @@ func (nb *Reader) Read(p []byte) (int, error) {
|
||||
}
|
||||
|
||||
// PutObject uploads the single upload to B2 backend by using *b2_upload_file* API, uploads upto 5GiB.
|
||||
func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, data *h2.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
func (l *b2Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
data := r.Reader
|
||||
|
||||
bkt, err := l.Bucket(ctx, bucket)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
@@ -653,7 +655,8 @@ func (l *b2Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket, uses B2's LargeFile upload API.
|
||||
func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *h2.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
|
||||
func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
|
||||
data := r.Reader
|
||||
bkt, err := l.Bucket(ctx, bucket)
|
||||
if err != nil {
|
||||
return pi, err
|
||||
@@ -681,7 +684,7 @@ func (l *b2Objects) PutObjectPart(ctx context.Context, bucket string, object str
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket, uses B2's LargeFile upload API.
|
||||
func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, err error) {
|
||||
func (l *b2Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) {
|
||||
bkt, err := l.Bucket(ctx, bucket)
|
||||
if err != nil {
|
||||
return lpi, err
|
||||
@@ -726,7 +729,7 @@ func (l *b2Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object, uses B2's LargeFile upload API.
|
||||
func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) {
|
||||
func (l *b2Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
|
||||
bkt, err := l.Bucket(ctx, bucket)
|
||||
if err != nil {
|
||||
return oi, err
|
||||
|
||||
@@ -40,7 +40,6 @@ import (
|
||||
miniogopolicy "github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
"github.com/minio/minio/pkg/policy/condition"
|
||||
|
||||
@@ -148,7 +147,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: gcsBackend,
|
||||
Usage: "Google Cloud Storage.",
|
||||
Usage: "Google Cloud Storage",
|
||||
Action: gcsGatewayMain,
|
||||
CustomHelpTemplate: gcsGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -207,11 +206,10 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
|
||||
gcs := &gcsGateway{
|
||||
client: client,
|
||||
projectID: g.projectID,
|
||||
ctx: ctx,
|
||||
}
|
||||
|
||||
// Start background process to cleanup old files in minio.sys.tmp
|
||||
go gcs.CleanupGCSMinioSysTmp()
|
||||
go gcs.CleanupGCSMinioSysTmp(ctx)
|
||||
return gcs, nil
|
||||
}
|
||||
|
||||
@@ -350,7 +348,6 @@ type gcsGateway struct {
|
||||
minio.GatewayUnsupported
|
||||
client *storage.Client
|
||||
projectID string
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// Returns projectID from the GOOGLE_APPLICATION_CREDENTIALS file.
|
||||
@@ -367,8 +364,8 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
|
||||
}
|
||||
|
||||
// Cleanup old files in minio.sys.tmp of the given bucket.
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) {
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err != nil {
|
||||
@@ -381,7 +378,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
}
|
||||
if time.Since(attrs.Updated) > gcsMultipartExpiry {
|
||||
// Delete files older than 2 weeks.
|
||||
err := l.client.Bucket(bucket).Object(attrs.Name).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Object(attrs.Name).Delete(ctx)
|
||||
if err != nil {
|
||||
reqInfo := &logger.ReqInfo{BucketName: bucket, ObjectName: attrs.Name}
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
@@ -393,9 +390,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
}
|
||||
|
||||
// Cleanup old files in minio.sys.tmp of all buckets.
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmp() {
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmp(ctx context.Context) {
|
||||
for {
|
||||
it := l.client.Buckets(l.ctx, l.projectID)
|
||||
it := l.client.Buckets(ctx, l.projectID)
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err != nil {
|
||||
@@ -405,7 +402,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() {
|
||||
}
|
||||
break
|
||||
}
|
||||
l.CleanupGCSMinioSysTmpBucket(attrs.Name)
|
||||
l.CleanupGCSMinioSysTmpBucket(ctx, attrs.Name)
|
||||
}
|
||||
// Run the cleanup loop every 1 day.
|
||||
time.Sleep(gcsCleanupInterval)
|
||||
@@ -432,7 +429,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
|
||||
location = "us"
|
||||
}
|
||||
|
||||
err := bkt.Create(l.ctx, l.projectID, &storage.BucketAttrs{
|
||||
err := bkt.Create(ctx, l.projectID, &storage.BucketAttrs{
|
||||
Location: location,
|
||||
})
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -441,7 +438,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
|
||||
|
||||
// GetBucketInfo - Get bucket metadata..
|
||||
func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) {
|
||||
attrs, err := l.client.Bucket(bucket).Attrs(l.ctx)
|
||||
attrs, err := l.client.Bucket(bucket).Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.BucketInfo{}, gcsToObjectError(err, bucket)
|
||||
@@ -455,7 +452,7 @@ func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.Bu
|
||||
|
||||
// ListBuckets lists all buckets under your project-id on GCS.
|
||||
func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
|
||||
it := l.client.Buckets(l.ctx, l.projectID)
|
||||
it := l.client.Buckets(ctx, l.projectID)
|
||||
|
||||
// Iterate and capture all the buckets.
|
||||
for {
|
||||
@@ -480,7 +477,7 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
|
||||
|
||||
// DeleteBucket delete a bucket on GCS.
|
||||
func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
itObject := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
itObject := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: "/",
|
||||
Versions: false,
|
||||
})
|
||||
@@ -510,7 +507,7 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
}
|
||||
if gcsMinioPathFound {
|
||||
// Remove minio.sys.tmp before deleting the bucket.
|
||||
itObject = l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
|
||||
itObject = l.client.Bucket(bucket).Objects(ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
|
||||
for {
|
||||
objAttrs, err := itObject.Next()
|
||||
if err == iterator.Done {
|
||||
@@ -520,14 +517,14 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err)
|
||||
}
|
||||
err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(l.ctx)
|
||||
err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
err := l.client.Bucket(bucket).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Delete(ctx)
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -562,7 +559,7 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
|
||||
return minio.ListObjectsInfo{}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: delimiter,
|
||||
Prefix: prefix,
|
||||
Versions: false,
|
||||
@@ -669,7 +666,7 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
|
||||
return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: delimiter,
|
||||
Prefix: prefix,
|
||||
Versions: false,
|
||||
@@ -770,7 +767,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string,
|
||||
func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -782,7 +779,7 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s
|
||||
// Calling ReadCompressed with true accomplishes that.
|
||||
object := l.client.Bucket(bucket).Object(key).ReadCompressed(true)
|
||||
|
||||
r, err := object.NewRangeReader(l.ctx, startOffset, length)
|
||||
r, err := object.NewRangeReader(ctx, startOffset, length)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, key)
|
||||
@@ -865,12 +862,12 @@ func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAt
|
||||
func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
attrs, err := l.client.Bucket(bucket).Object(object).Attrs(l.ctx)
|
||||
attrs, err := l.client.Bucket(bucket).Object(object).Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, object)
|
||||
@@ -880,17 +877,19 @@ func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object st
|
||||
}
|
||||
|
||||
// PutObject - Create a new object with the incoming data,
|
||||
func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
|
||||
func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
|
||||
data := r.Reader
|
||||
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
object := l.client.Bucket(bucket).Object(key)
|
||||
|
||||
w := object.NewWriter(l.ctx)
|
||||
w := object.NewWriter(ctx)
|
||||
// Disable "chunked" uploading in GCS client if the size of the data to be uploaded is below
|
||||
// the current chunk-size of the writer. This avoids an unnecessary memory allocation.
|
||||
if data.Size() < int64(w.ChunkSize) {
|
||||
@@ -908,7 +907,7 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, d
|
||||
// Close the object writer upon success.
|
||||
w.Close()
|
||||
|
||||
attrs, err := object.Attrs(l.ctx)
|
||||
attrs, err := object.Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -927,7 +926,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
|
||||
copier := dst.CopierFrom(src)
|
||||
applyMetadataToGCSAttrs(srcInfo.UserDefined, &copier.ObjectAttrs)
|
||||
|
||||
attrs, err := copier.Run(l.ctx)
|
||||
attrs, err := copier.Run(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, destBucket, destObject)
|
||||
@@ -938,7 +937,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
|
||||
|
||||
// DeleteObject - Deletes a blob in bucket
|
||||
func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error {
|
||||
err := l.client.Bucket(bucket).Object(object).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Object(object).Delete(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, object)
|
||||
@@ -955,7 +954,7 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
|
||||
// generate name for part zero
|
||||
meta := gcsMultipartMetaName(uploadID)
|
||||
|
||||
w := l.client.Bucket(bucket).Object(meta).NewWriter(l.ctx)
|
||||
w := l.client.Bucket(bucket).Object(meta).NewWriter(ctx)
|
||||
defer w.Close()
|
||||
|
||||
applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
|
||||
@@ -1034,8 +1033,7 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr
|
||||
UploadID: components[3],
|
||||
Initiated: attrs.Created,
|
||||
}
|
||||
uploads = []minio.MultipartInfo{upload}
|
||||
break
|
||||
uploads = append(uploads, upload)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1055,13 +1053,14 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr
|
||||
// Checks if minio.sys.tmp/multipart/v1/<upload-id>/gcs.json exists, returns
|
||||
// an object layer compatible error upon any error.
|
||||
func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error {
|
||||
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx)
|
||||
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(ctx)
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, key, uploadID)
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, data *hash.Reader, opts minio.ObjectOptions) (minio.PartInfo, error) {
|
||||
func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key string, uploadID string, partNumber int, r *minio.PutObjReader, opts minio.ObjectOptions) (minio.PartInfo, error) {
|
||||
data := r.Reader
|
||||
if err := l.checkUploadIDExists(ctx, bucket, key, uploadID); err != nil {
|
||||
return minio.PartInfo{}, err
|
||||
}
|
||||
@@ -1071,7 +1070,7 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
|
||||
etag = minio.GenETag()
|
||||
}
|
||||
object := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, partNumber, etag))
|
||||
w := object.NewWriter(l.ctx)
|
||||
w := object.NewWriter(ctx)
|
||||
// Disable "chunked" uploading in GCS client. If enabled, it can cause a corner case
|
||||
// where it tries to upload 0 bytes in the last chunk and get error from server.
|
||||
w.ChunkSize = 0
|
||||
@@ -1121,7 +1120,7 @@ func gcsGetPartInfo(ctx context.Context, attrs *storage.ObjectAttrs) (minio.Part
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int) (minio.ListPartsInfo, error) {
|
||||
func (l *gcsGateway) ListObjectParts(ctx context.Context, bucket string, key string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (minio.ListPartsInfo, error) {
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Prefix: path.Join(gcsMinioMultipartPathV1, uploadID),
|
||||
})
|
||||
@@ -1184,7 +1183,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
|
||||
prefix := fmt.Sprintf("%s/%s/", gcsMinioMultipartPathV1, uploadID)
|
||||
|
||||
// iterate through all parts and delete them
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: prefix, Versions: false})
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: prefix, Versions: false})
|
||||
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
@@ -1198,7 +1197,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
|
||||
|
||||
object := l.client.Bucket(bucket).Object(attrs.Name)
|
||||
// Ignore the error as parallel AbortMultipartUpload might have deleted it.
|
||||
object.Delete(l.ctx)
|
||||
object.Delete(ctx)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1216,17 +1215,17 @@ func (l *gcsGateway) AbortMultipartUpload(ctx context.Context, bucket string, ke
|
||||
// Note that there is a limit (currently 32) to the number of components that can
|
||||
// be composed in a single operation. There is a per-project rate limit (currently 200)
|
||||
// to the number of source objects you can compose per second.
|
||||
func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart) (minio.ObjectInfo, error) {
|
||||
func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string, key string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
|
||||
meta := gcsMultipartMetaName(uploadID)
|
||||
object := l.client.Bucket(bucket).Object(meta)
|
||||
|
||||
partZeroAttrs, err := object.Attrs(l.ctx)
|
||||
partZeroAttrs, err := object.Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key, uploadID)
|
||||
}
|
||||
|
||||
r, err := object.NewReader(l.ctx)
|
||||
r, err := object.NewReader(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -1257,7 +1256,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
for i, uploadedPart := range uploadedParts {
|
||||
parts = append(parts, l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID,
|
||||
uploadedPart.PartNumber, uploadedPart.ETag)))
|
||||
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(l.ctx)
|
||||
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(ctx)
|
||||
if pErr != nil {
|
||||
logger.LogIf(ctx, pErr)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(pErr, bucket, key, uploadID)
|
||||
@@ -1303,7 +1302,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
composer.ContentType = partZeroAttrs.ContentType
|
||||
composer.Metadata = partZeroAttrs.Metadata
|
||||
|
||||
if _, err = composer.Run(l.ctx); err != nil {
|
||||
if _, err = composer.Run(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
}
|
||||
@@ -1320,7 +1319,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
composer.ContentDisposition = partZeroAttrs.ContentDisposition
|
||||
composer.ContentLanguage = partZeroAttrs.ContentLanguage
|
||||
composer.Metadata = partZeroAttrs.Metadata
|
||||
attrs, err := composer.Run(l.ctx)
|
||||
attrs, err := composer.Run(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -1360,7 +1359,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
|
||||
acl := l.client.Bucket(bucket).ACL()
|
||||
if policies[0].Policy == miniogopolicy.BucketPolicyNone {
|
||||
if err := acl.Delete(l.ctx, storage.AllUsers); err != nil {
|
||||
if err := acl.Delete(ctx, storage.AllUsers); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1378,7 +1377,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
return minio.NotImplemented{}
|
||||
}
|
||||
|
||||
if err := acl.Set(l.ctx, storage.AllUsers, role); err != nil {
|
||||
if err := acl.Set(ctx, storage.AllUsers, role); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1388,7 +1387,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
|
||||
// GetBucketPolicy - Get policy on bucket
|
||||
func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
|
||||
rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
|
||||
rules, err := l.client.Bucket(bucket).ACL().List(ctx)
|
||||
if err != nil {
|
||||
return nil, gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1447,7 +1446,7 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
|
||||
// DeleteBucketPolicy - Delete all policies on bucket
|
||||
func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
||||
// This only removes the storage.AllUsers policies
|
||||
if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
|
||||
if err := l.client.Bucket(bucket).ACL().Delete(ctx, storage.AllUsers); err != nil {
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,6 @@ import (
|
||||
minio "github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// stor is a namespace within manta where you store any documents that are deemed as private
|
||||
@@ -109,7 +108,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: mantaBackend,
|
||||
Usage: "Manta Object Storage.",
|
||||
Usage: "Manta Object Storage",
|
||||
Action: mantaGatewayMain,
|
||||
CustomHelpTemplate: mantaGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -608,7 +607,8 @@ func (d dummySeeker) Seek(offset int64, whence int) (int64, error) {
|
||||
// CreateBlockBlobFromReader.
|
||||
//
|
||||
// https://apidocs.joyent.com/manta/api.html#PutObject
|
||||
func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
func (t *tritonObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
data := r.Reader
|
||||
if err = t.client.Objects().Put(ctx, &storage.PutObjectInput{
|
||||
ContentLength: uint64(data.Size()),
|
||||
ObjectPath: path.Join(mantaRoot, bucket, object),
|
||||
|
||||
@@ -74,7 +74,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: nasBackend,
|
||||
Usage: "Network-attached storage (NAS).",
|
||||
Usage: "Network-attached storage (NAS)",
|
||||
Action: nasGatewayMain,
|
||||
CustomHelpTemplate: nasGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -108,7 +108,7 @@ func (g *NAS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &nasObjects{newObject.(*minio.FSObjects)}, nil
|
||||
return &nasObjects{newObject}, nil
|
||||
}
|
||||
|
||||
// Production - nas gateway is production ready.
|
||||
@@ -116,17 +116,12 @@ func (g *NAS) Production() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this gateway.
|
||||
func (n *nasObjects) IsListenBucketSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// nasObjects implements gateway for Minio and S3 compatible object storage servers.
|
||||
type nasObjects struct {
|
||||
*minio.FSObjects
|
||||
}
|
||||
|
||||
// IsNotificationSupported returns whether notifications are applicable for this layer.
|
||||
func (l *nasObjects) IsNotificationSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsCompressionSupported returns whether compression is applicable for this layer.
|
||||
func (l *nasObjects) IsCompressionSupported() bool {
|
||||
return false
|
||||
minio.ObjectLayer
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: "oss",
|
||||
Usage: "Alibaba Cloud (Aliyun) Object Storage Service (OSS).",
|
||||
Usage: "Alibaba Cloud (Aliyun) Object Storage Service (OSS)",
|
||||
Action: ossGatewayMain,
|
||||
CustomHelpTemplate: ossGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -655,7 +655,9 @@ func ossPutObject(ctx context.Context, client *oss.Client, bucket, object string
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data.
|
||||
func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
func (l *ossObjects) PutObject(ctx context.Context, bucket, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
data := r.Reader
|
||||
|
||||
return ossPutObject(ctx, l.Client, bucket, object, data, metadata)
|
||||
}
|
||||
|
||||
@@ -773,7 +775,8 @@ func (l *ossObjects) NewMultipartUpload(ctx context.Context, bucket, object stri
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket.
|
||||
func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
|
||||
func (l *ossObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, err error) {
|
||||
data := r.Reader
|
||||
bkt, err := l.Client.Bucket(bucket)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -883,7 +886,7 @@ func (l *ossObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, d
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int) (lpi minio.ListPartsInfo, err error) {
|
||||
func (l *ossObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, err error) {
|
||||
lupr, err := ossListObjectParts(l.Client, bucket, object, uploadID, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -914,7 +917,7 @@ func (l *ossObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object.
|
||||
func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, err error) {
|
||||
func (l *ossObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, err error) {
|
||||
client := l.Client
|
||||
bkt, err := client.Bucket(bucket)
|
||||
if err != nil {
|
||||
|
||||
@@ -0,0 +1,238 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
minio "github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
var (
|
||||
errGWMetaNotFound = errors.New("dare.meta file not found")
|
||||
errGWMetaInvalidFormat = errors.New("dare.meta format is invalid")
|
||||
)
|
||||
|
||||
// A gwMetaV1 represents `gw.json` metadata header.
|
||||
type gwMetaV1 struct {
|
||||
Version string `json:"version"` // Version of the current `gw.json`.
|
||||
Format string `json:"format"` // Format of the current `gw.json`.
|
||||
Stat minio.StatInfo `json:"stat"` // Stat of the current object `gw.json`.
|
||||
ETag string `json:"etag"` // ETag of the current object
|
||||
|
||||
// Metadata map for current object `gw.json`.
|
||||
Meta map[string]string `json:"meta,omitempty"`
|
||||
// Captures all the individual object `gw.json`.
|
||||
Parts []minio.ObjectPartInfo `json:"parts,omitempty"`
|
||||
}
|
||||
|
||||
// Gateway metadata constants.
|
||||
const (
|
||||
// Gateway meta version.
|
||||
gwMetaVersion = "1.0.0"
|
||||
|
||||
// Gateway meta version.
|
||||
gwMetaVersion100 = "1.0.0"
|
||||
|
||||
// Gateway meta format string.
|
||||
gwMetaFormat = "gw"
|
||||
|
||||
// Add new constants here.
|
||||
)
|
||||
|
||||
// newGWMetaV1 - initializes new gwMetaV1, adds version.
|
||||
func newGWMetaV1() (gwMeta gwMetaV1) {
|
||||
gwMeta = gwMetaV1{}
|
||||
gwMeta.Version = gwMetaVersion
|
||||
gwMeta.Format = gwMetaFormat
|
||||
return gwMeta
|
||||
}
|
||||
|
||||
// IsValid - tells if the format is sane by validating the version
|
||||
// string, format fields.
|
||||
func (m gwMetaV1) IsValid() bool {
|
||||
return ((m.Version == gwMetaVersion || m.Version == gwMetaVersion100) &&
|
||||
m.Format == gwMetaFormat)
|
||||
}
|
||||
|
||||
// Converts metadata to object info.
|
||||
func (m gwMetaV1) ToObjectInfo(bucket, object string) minio.ObjectInfo {
|
||||
filterKeys := append([]string{
|
||||
"ETag",
|
||||
"Content-Length",
|
||||
"Last-Modified",
|
||||
"Content-Type",
|
||||
}, defaultFilterKeys...)
|
||||
objInfo := minio.ObjectInfo{
|
||||
IsDir: false,
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
Size: m.Stat.Size,
|
||||
ModTime: m.Stat.ModTime,
|
||||
ContentType: m.Meta["content-type"],
|
||||
ContentEncoding: m.Meta["content-encoding"],
|
||||
ETag: minio.CanonicalizeETag(m.ETag),
|
||||
UserDefined: minio.CleanMetadataKeys(m.Meta, filterKeys...),
|
||||
Parts: m.Parts,
|
||||
}
|
||||
|
||||
if sc, ok := m.Meta["x-amz-storage-class"]; ok {
|
||||
objInfo.StorageClass = sc
|
||||
}
|
||||
// Success.
|
||||
return objInfo
|
||||
}
|
||||
|
||||
// ObjectToPartOffset - translate offset of an object to offset of its individual part.
|
||||
func (m gwMetaV1) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) {
|
||||
if offset == 0 {
|
||||
// Special case - if offset is 0, then partIndex and partOffset are always 0.
|
||||
return 0, 0, nil
|
||||
}
|
||||
partOffset = offset
|
||||
// Seek until object offset maps to a particular part offset.
|
||||
for i, part := range m.Parts {
|
||||
partIndex = i
|
||||
// Offset is smaller than size we have reached the proper part offset.
|
||||
if partOffset < part.Size {
|
||||
return partIndex, partOffset, nil
|
||||
}
|
||||
// Continue to towards the next part.
|
||||
partOffset -= part.Size
|
||||
}
|
||||
logger.LogIf(ctx, minio.InvalidRange{})
|
||||
// Offset beyond the size of the object return InvalidRange.
|
||||
return 0, 0, minio.InvalidRange{}
|
||||
}
|
||||
|
||||
// parses gateway metadata stat info from metadata json
|
||||
func parseGWStat(gwMetaBuf []byte) (si minio.StatInfo, e error) {
|
||||
// obtain stat info.
|
||||
stat := minio.StatInfo{}
|
||||
// fetching modTime.
|
||||
modTime, err := time.Parse(time.RFC3339, gjson.GetBytes(gwMetaBuf, "stat.modTime").String())
|
||||
if err != nil {
|
||||
return si, err
|
||||
}
|
||||
stat.ModTime = modTime
|
||||
// obtain Stat.Size .
|
||||
stat.Size = gjson.GetBytes(gwMetaBuf, "stat.size").Int()
|
||||
return stat, nil
|
||||
}
|
||||
|
||||
// parses gateway metadata version from metadata json
|
||||
func parseGWVersion(gwMetaBuf []byte) string {
|
||||
return gjson.GetBytes(gwMetaBuf, "version").String()
|
||||
}
|
||||
|
||||
// parses gateway ETag from metadata json
|
||||
func parseGWETag(gwMetaBuf []byte) string {
|
||||
return gjson.GetBytes(gwMetaBuf, "etag").String()
|
||||
}
|
||||
|
||||
// parses gateway metadata format from metadata json
|
||||
func parseGWFormat(gwMetaBuf []byte) string {
|
||||
return gjson.GetBytes(gwMetaBuf, "format").String()
|
||||
}
|
||||
|
||||
// parses gateway metadata json to get list of ObjectPartInfo
|
||||
func parseGWParts(gwMetaBuf []byte) []minio.ObjectPartInfo {
|
||||
// Parse the GW Parts.
|
||||
partsResult := gjson.GetBytes(gwMetaBuf, "parts").Array()
|
||||
partInfo := make([]minio.ObjectPartInfo, len(partsResult))
|
||||
for i, p := range partsResult {
|
||||
info := minio.ObjectPartInfo{}
|
||||
info.Number = int(p.Get("number").Int())
|
||||
info.Name = p.Get("name").String()
|
||||
info.ETag = p.Get("etag").String()
|
||||
info.Size = p.Get("size").Int()
|
||||
partInfo[i] = info
|
||||
}
|
||||
return partInfo
|
||||
}
|
||||
|
||||
// parses gateway metadata json to get the metadata map
|
||||
func parseGWMetaMap(gwMetaBuf []byte) map[string]string {
|
||||
// Get gwMetaV1.Meta map.
|
||||
metaMapResult := gjson.GetBytes(gwMetaBuf, "meta").Map()
|
||||
metaMap := make(map[string]string)
|
||||
for key, valResult := range metaMapResult {
|
||||
metaMap[key] = valResult.String()
|
||||
}
|
||||
return metaMap
|
||||
}
|
||||
|
||||
// Constructs GWMetaV1 using `gjson` lib to retrieve each field.
|
||||
func gwMetaUnmarshalJSON(ctx context.Context, gwMetaBuf []byte) (gwMeta gwMetaV1, e error) {
|
||||
// obtain version.
|
||||
gwMeta.Version = parseGWVersion(gwMetaBuf)
|
||||
// obtain format.
|
||||
gwMeta.Format = parseGWFormat(gwMetaBuf)
|
||||
// Parse gwMetaV1.Stat .
|
||||
stat, err := parseGWStat(gwMetaBuf)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gwMeta, err
|
||||
}
|
||||
gwMeta.ETag = parseGWETag(gwMetaBuf)
|
||||
gwMeta.Stat = stat
|
||||
|
||||
// Parse the GW Parts.
|
||||
gwMeta.Parts = parseGWParts(gwMetaBuf)
|
||||
// parse gwMetaV1.
|
||||
gwMeta.Meta = parseGWMetaMap(gwMetaBuf)
|
||||
|
||||
return gwMeta, nil
|
||||
}
|
||||
|
||||
// readGWMeta reads `dare.meta` and returns back GW metadata structure.
|
||||
func readGWMetadata(ctx context.Context, buf bytes.Buffer) (gwMeta gwMetaV1, err error) {
|
||||
if buf.Len() == 0 {
|
||||
return gwMetaV1{}, errGWMetaNotFound
|
||||
}
|
||||
gwMeta, err = gwMetaUnmarshalJSON(ctx, buf.Bytes())
|
||||
if err != nil {
|
||||
return gwMetaV1{}, err
|
||||
}
|
||||
if !gwMeta.IsValid() {
|
||||
return gwMetaV1{}, errGWMetaInvalidFormat
|
||||
}
|
||||
// Return structured `dare.meta`.
|
||||
return gwMeta, nil
|
||||
}
|
||||
|
||||
// getGWMetadata - unmarshals dare.meta into a *minio.PutObjReader
|
||||
func getGWMetadata(ctx context.Context, bucket, prefix string, gwMeta gwMetaV1) (*minio.PutObjReader, error) {
|
||||
// Marshal json.
|
||||
metadataBytes, err := json.Marshal(&gwMeta)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, err
|
||||
}
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(metadataBytes), int64(len(metadataBytes)), "", "", int64(len(metadataBytes)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return minio.NewPutObjReader(hashReader, nil, nil), nil
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests for GW metadata format validity.
|
||||
func TestGWMetaFormatValid(t *testing.T) {
|
||||
tests := []struct {
|
||||
name int
|
||||
version string
|
||||
format string
|
||||
want bool
|
||||
}{
|
||||
{1, "123", "fs", false},
|
||||
{2, "123", gwMetaFormat, false},
|
||||
{3, gwMetaVersion, "test", false},
|
||||
{4, gwMetaVersion100, "hello", false},
|
||||
{5, gwMetaVersion, gwMetaFormat, true},
|
||||
{6, gwMetaVersion100, gwMetaFormat, true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
m := newGWMetaV1()
|
||||
m.Version = tt.version
|
||||
m.Format = tt.format
|
||||
if got := m.IsValid(); got != tt.want {
|
||||
t.Errorf("Test %d: Expected %v but received %v", tt.name, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests for reading GW metadata info.
|
||||
func TestReadGWMetadata(t *testing.T) {
|
||||
tests := []struct {
|
||||
metaStr string
|
||||
pass bool
|
||||
}{
|
||||
{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "132", "modTime": "2018-08-31T22:25:39.23626461Z" }}}`, true},
|
||||
{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "132", "modTime": "0000-00-00T00:00:00.00000000Z" }}}`, false},
|
||||
{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "5242880", "modTime": "2018-08-31T22:25:39.23626461Z" }},"meta":{"content-type":"application/octet-stream","etag":"57c743902b2fc8eea6ba3bb4fc58c8e8"},"parts":[{"number":1,"name":"part.1","etag":"","size":5242880}]}}`, true},
|
||||
{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "68190720", "modTime": "2018-08-31T22:25:39.23626461Z" }},"meta":{"X-Minio-Internal-Encrypted-Multipart":"","X-Minio-Internal-Server-Side-Encryption-Iv":"kdbOcKdXD3Sew8tOiHe5eI9xkX1oQ2W9JURz0oslCZA=","X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":"DAREv2-HMAC-SHA256","X-Minio-Internal-Server-Side-Encryption-Sealed-Key":"IAAfAMfqKrxMXC9LuiI7ENP+p0xArepzAiIeB/MftFp7Xmq2OzDkKlmNbj5RKI89RrjiAbOVLSSEMvqQsrIrTQ==","content-type":"text/plain; charset=utf-8","etag":"2b137fa4ab80126af54623b010c98de6-2"},"parts":[{"number":1,"name":"part.1","etag":"c5cac075eefdab801a5198812f51b36e","size":67141632},{"number":2,"name":"part.2","etag":"ccdf4b774bc3be8eef9a8987309e8171","size":1049088}]}`, true},
|
||||
{`{"version": "` + gwMetaVersion + `", "format":"` + gwMetaFormat + `", {"stat": {"size": "68190720", "modTime": "2018-08-31T22:25:39.23626461Z" }},"meta":{"X-Minio-Internal-Encrypted-Multipart":"","X-Minio-Internal-Server-Side-Encryption-Iv":"kdbOcKdXD3Sew8tOiHe5eI9xkX1oQ2W9JURz0oslCZA=","X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":"DAREv2-HMAC-SHA256","X-Minio-Internal-Server-Side-Encryption-Sealed-Key":"IAAfAMfqKrxMXC9LuiI7ENP+p0xArepzAiIeB/MftFp7Xmq2OzDkKlmNbj5RKI89RrjiAbOVLSSEMvqQsrIrTQ==","content-type":"text/plain; charset=utf-8","etag":"2b137fa4ab80126af54623b010c98de6-2"},"parts":"123"}`, true},
|
||||
}
|
||||
|
||||
for i, tt := range tests {
|
||||
buf := bytes.NewBufferString(tt.metaStr)
|
||||
m, err := readGWMetadata(context.Background(), *buf)
|
||||
if err != nil && tt.pass {
|
||||
t.Errorf("Test %d: Expected parse gw metadata to succeed, but failed", i)
|
||||
}
|
||||
if err == nil && !tt.pass {
|
||||
t.Errorf("Test %d: Expected parse gw metadata to succeed, but failed", i)
|
||||
}
|
||||
if err == nil {
|
||||
if m.Version != gwMetaVersion {
|
||||
t.Errorf("Test %d: Expected version %s, but failed with %s", i, gwMetaVersion, m.Version)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,778 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
// name of custom multipart metadata file for s3 backend.
|
||||
gwdareMetaJSON string = "dare.meta"
|
||||
|
||||
// name of temporary per part metadata file
|
||||
gwpartMetaJSON string = "part.meta"
|
||||
// custom multipart files are stored under the defaultMinioGWPrefix
|
||||
defaultMinioGWPrefix = ".minio"
|
||||
defaultGWContentFileName = "data"
|
||||
slashSeparator = "/"
|
||||
)
|
||||
|
||||
// s3EncObjects is a wrapper around s3Objects and implements gateway calls for
|
||||
// custom large objects encrypted at the gateway
|
||||
type s3EncObjects struct {
|
||||
s3Objects
|
||||
}
|
||||
|
||||
/*
|
||||
NOTE:
|
||||
Custom gateway encrypted objects are stored on backend as follows:
|
||||
obj/.minio/data <= encrypted content
|
||||
obj/.minio/dare.meta <= metadata
|
||||
|
||||
When a multipart upload operation is in progress, the metadata set during
|
||||
NewMultipartUpload is stored in obj/.minio/uploadID/dare.meta and each
|
||||
UploadPart operation saves additional state of the part's encrypted ETag and
|
||||
encrypted size in obj/.minio/uploadID/part1/part.meta
|
||||
|
||||
All the part metadata and temp dare.meta are cleaned up when upload completes
|
||||
*/
|
||||
|
||||
// ListObjects lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3EncObjects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) {
|
||||
var continuationToken, startAfter string
|
||||
res, err := l.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, maxKeys, false, startAfter)
|
||||
if err != nil {
|
||||
return loi, err
|
||||
}
|
||||
loi.IsTruncated = res.IsTruncated
|
||||
loi.NextMarker = res.NextContinuationToken
|
||||
loi.Objects = res.Objects
|
||||
loi.Prefixes = res.Prefixes
|
||||
return loi, nil
|
||||
|
||||
}
|
||||
|
||||
// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
|
||||
|
||||
var objects []minio.ObjectInfo
|
||||
var prefixes []string
|
||||
var isTruncated bool
|
||||
|
||||
// filter out objects that contain a .minio prefix, but is not a dare.meta metadata file.
|
||||
for {
|
||||
loi, e = l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, fetchOwner, startAfter)
|
||||
if e != nil {
|
||||
return loi, minio.ErrorRespToObjectError(e, bucket)
|
||||
}
|
||||
for _, obj := range loi.Objects {
|
||||
startAfter = obj.Name
|
||||
continuationToken = loi.NextContinuationToken
|
||||
isTruncated = loi.IsTruncated
|
||||
|
||||
if !isGWObject(obj.Name) {
|
||||
continue
|
||||
}
|
||||
// get objectname and ObjectInfo from the custom metadata file
|
||||
if strings.HasSuffix(obj.Name, gwdareMetaJSON) {
|
||||
objSlice := strings.Split(obj.Name, slashSeparator+defaultMinioGWPrefix)
|
||||
gwMeta, e := l.getGWMetadata(ctx, bucket, getDareMetaPath(objSlice[0]))
|
||||
if e != nil {
|
||||
continue
|
||||
}
|
||||
oInfo := gwMeta.ToObjectInfo(bucket, objSlice[0])
|
||||
objects = append(objects, oInfo)
|
||||
} else {
|
||||
objects = append(objects, obj)
|
||||
}
|
||||
if len(objects) > maxKeys {
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, p := range loi.Prefixes {
|
||||
objName := strings.TrimSuffix(p, slashSeparator)
|
||||
gm, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(objName))
|
||||
// if prefix is actually a custom multi-part object, append it to objects
|
||||
if err == nil {
|
||||
objects = append(objects, gm.ToObjectInfo(bucket, objName))
|
||||
continue
|
||||
}
|
||||
isPrefix := l.isPrefix(ctx, bucket, p, fetchOwner, startAfter)
|
||||
if isPrefix {
|
||||
prefixes = append(prefixes, p)
|
||||
}
|
||||
}
|
||||
if (len(objects) > maxKeys) || !loi.IsTruncated {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
loi.IsTruncated = isTruncated
|
||||
loi.ContinuationToken = continuationToken
|
||||
loi.Objects = make([]minio.ObjectInfo, 0)
|
||||
loi.Prefixes = make([]string, 0)
|
||||
|
||||
for _, obj := range objects {
|
||||
loi.NextContinuationToken = obj.Name
|
||||
loi.Objects = append(loi.Objects, obj)
|
||||
}
|
||||
for _, pfx := range prefixes {
|
||||
if pfx != prefix {
|
||||
loi.Prefixes = append(loi.Prefixes, pfx)
|
||||
}
|
||||
}
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
// isGWObject returns true if it is a custom object
|
||||
func isGWObject(objName string) bool {
|
||||
isEncrypted := strings.Contains(objName, defaultMinioGWPrefix)
|
||||
if !isEncrypted {
|
||||
return true
|
||||
}
|
||||
// ignore temp part.meta files
|
||||
if strings.Contains(objName, gwpartMetaJSON) {
|
||||
return false
|
||||
}
|
||||
|
||||
pfxSlice := strings.Split(objName, slashSeparator)
|
||||
var i1, i2 int
|
||||
for i := len(pfxSlice) - 1; i >= 0; i-- {
|
||||
p := pfxSlice[i]
|
||||
if p == defaultMinioGWPrefix {
|
||||
i1 = i
|
||||
}
|
||||
if p == gwdareMetaJSON {
|
||||
i2 = i
|
||||
}
|
||||
if i1 > 0 && i2 > 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
// incomplete uploads would have a uploadID between defaultMinioGWPrefix and gwdareMetaJSON
|
||||
return i2 > 0 && i1 > 0 && i2-i1 == 1
|
||||
}
|
||||
|
||||
// isPrefix returns true if prefix exists and is not an incomplete multipart upload entry
|
||||
func (l *s3EncObjects) isPrefix(ctx context.Context, bucket, prefix string, fetchOwner bool, startAfter string) bool {
|
||||
var continuationToken, delimiter string
|
||||
|
||||
for {
|
||||
loi, e := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, fetchOwner, startAfter)
|
||||
if e != nil {
|
||||
return false
|
||||
}
|
||||
for _, obj := range loi.Objects {
|
||||
if isGWObject(obj.Name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
continuationToken = loi.NextContinuationToken
|
||||
if !loi.IsTruncated {
|
||||
break
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// GetObject reads an object from S3. Supports additional
|
||||
// parameters like offset and length which are synonymous with
|
||||
// HTTP Range requests.
|
||||
func (l *s3EncObjects) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
|
||||
return l.getObject(ctx, bucket, key, startOffset, length, writer, etag, opts)
|
||||
}
|
||||
|
||||
func (l *s3EncObjects) isGWEncrypted(ctx context.Context, bucket, object string) bool {
|
||||
_, err := l.s3Objects.GetObjectInfo(ctx, bucket, getDareMetaPath(object), minio.ObjectOptions{})
|
||||
return err == nil
|
||||
}
|
||||
|
||||
// getDaremetadata fetches dare.meta from s3 backend and marshals into a structured format.
|
||||
func (l *s3EncObjects) getGWMetadata(ctx context.Context, bucket, metaFileName string) (m gwMetaV1, err error) {
|
||||
oi, err1 := l.s3Objects.GetObjectInfo(ctx, bucket, metaFileName, minio.ObjectOptions{})
|
||||
if err1 != nil {
|
||||
return m, err1
|
||||
}
|
||||
var buffer bytes.Buffer
|
||||
err = l.s3Objects.GetObject(ctx, bucket, metaFileName, 0, oi.Size, &buffer, oi.ETag, minio.ObjectOptions{})
|
||||
if err != nil {
|
||||
return m, err
|
||||
}
|
||||
return readGWMetadata(ctx, buffer)
|
||||
}
|
||||
|
||||
// writes dare metadata to the s3 backend
|
||||
func (l *s3EncObjects) writeGWMetadata(ctx context.Context, bucket, metaFileName string, m gwMetaV1, o minio.ObjectOptions) error {
|
||||
reader, err := getGWMetadata(ctx, bucket, metaFileName, m)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
_, err = l.s3Objects.PutObject(ctx, bucket, metaFileName, reader, map[string]string{}, o)
|
||||
return err
|
||||
}
|
||||
|
||||
// returns path of temporary metadata json file for the upload
|
||||
func getTmpDareMetaPath(object, uploadID string) string {
|
||||
return path.Join(getGWMetaPath(object), uploadID, gwdareMetaJSON)
|
||||
}
|
||||
|
||||
// returns path of metadata json file for encrypted objects
|
||||
func getDareMetaPath(object string) string {
|
||||
return path.Join(getGWMetaPath(object), gwdareMetaJSON)
|
||||
}
|
||||
|
||||
// returns path of temporary part metadata file for multipart uploads
|
||||
func getPartMetaPath(object, uploadID string, partID int) string {
|
||||
return path.Join(object, defaultMinioGWPrefix, uploadID, strconv.Itoa(partID), gwpartMetaJSON)
|
||||
}
|
||||
|
||||
// deletes the custom dare metadata file saved at the backend
|
||||
func (l *s3EncObjects) deleteGWMetadata(ctx context.Context, bucket, metaFileName string) error {
|
||||
return l.s3Objects.DeleteObject(ctx, bucket, metaFileName)
|
||||
}
|
||||
|
||||
func (l *s3EncObjects) getObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
|
||||
var o minio.ObjectOptions
|
||||
if minio.GlobalGatewaySSE.SSEC() {
|
||||
o = opts
|
||||
}
|
||||
dmeta, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(key))
|
||||
if err != nil {
|
||||
// unencrypted content
|
||||
return l.s3Objects.GetObject(ctx, bucket, key, startOffset, length, writer, etag, o)
|
||||
}
|
||||
if startOffset < 0 {
|
||||
logger.LogIf(ctx, minio.InvalidRange{})
|
||||
}
|
||||
|
||||
// For negative length read everything.
|
||||
if length < 0 {
|
||||
length = dmeta.Stat.Size - startOffset
|
||||
}
|
||||
// Reply back invalid range if the input offset and length fall out of range.
|
||||
if startOffset > dmeta.Stat.Size || startOffset+length > dmeta.Stat.Size {
|
||||
logger.LogIf(ctx, minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size})
|
||||
return minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size}
|
||||
}
|
||||
// Get start part index and offset.
|
||||
_, partOffset, err := dmeta.ObjectToPartOffset(ctx, startOffset)
|
||||
if err != nil {
|
||||
return minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size}
|
||||
}
|
||||
|
||||
// Calculate endOffset according to length
|
||||
endOffset := startOffset
|
||||
if length > 0 {
|
||||
endOffset += length - 1
|
||||
}
|
||||
|
||||
// Get last part index to read given length.
|
||||
if _, _, err := dmeta.ObjectToPartOffset(ctx, endOffset); err != nil {
|
||||
return minio.InvalidRange{OffsetBegin: startOffset, OffsetEnd: length, ResourceSize: dmeta.Stat.Size}
|
||||
}
|
||||
return l.s3Objects.GetObject(ctx, bucket, key, partOffset, endOffset, writer, dmeta.ETag, o)
|
||||
}
|
||||
|
||||
// GetObjectNInfo - returns object info and locked object ReadCloser
|
||||
func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *minio.HTTPRangeSpec, h http.Header, lockType minio.LockType, o minio.ObjectOptions) (gr *minio.GetObjectReader, err error) {
|
||||
var opts minio.ObjectOptions
|
||||
if minio.GlobalGatewaySSE.SSEC() {
|
||||
opts = o
|
||||
}
|
||||
objInfo, err := l.GetObjectInfo(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return l.s3Objects.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
||||
}
|
||||
objInfo.UserDefined = minio.CleanMinioInternalMetadataKeys(objInfo.UserDefined)
|
||||
fn, off, length, err := minio.NewGetObjectReader(rs, objInfo)
|
||||
if err != nil {
|
||||
return nil, minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
if l.isGWEncrypted(ctx, bucket, object) {
|
||||
object = getGWContentPath(object)
|
||||
}
|
||||
pr, pw := io.Pipe()
|
||||
go func() {
|
||||
err := l.getObject(ctx, bucket, object, off, length, pw, objInfo.ETag, opts)
|
||||
pw.CloseWithError(err)
|
||||
}()
|
||||
|
||||
// Setup cleanup function to cause the above go-routine to
|
||||
// exit in case of partial read
|
||||
pipeCloser := func() { pr.Close() }
|
||||
return fn(pr, h, pipeCloser)
|
||||
}
|
||||
|
||||
// GetObjectInfo reads object info and replies back ObjectInfo
|
||||
// For custom gateway encrypted large objects, the ObjectInfo is retrieved from the dare.meta file.
|
||||
func (l *s3EncObjects) GetObjectInfo(ctx context.Context, bucket string, object string, o minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
var opts minio.ObjectOptions
|
||||
if minio.GlobalGatewaySSE.SSEC() {
|
||||
opts = o
|
||||
}
|
||||
|
||||
gwMeta, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(object))
|
||||
if err != nil {
|
||||
return l.s3Objects.GetObjectInfo(ctx, bucket, object, opts)
|
||||
}
|
||||
return gwMeta.ToObjectInfo(bucket, object), nil
|
||||
}
|
||||
|
||||
// CopyObject copies an object from source bucket to a destination bucket.
|
||||
func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, s, d minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
cpSrcDstSame := strings.EqualFold(path.Join(srcBucket, srcObject), path.Join(dstBucket, dstObject))
|
||||
if cpSrcDstSame {
|
||||
var gwMeta gwMetaV1
|
||||
if s.ServerSideEncryption != nil && d.ServerSideEncryption != nil &&
|
||||
((s.ServerSideEncryption.Type() == encrypt.SSEC && d.ServerSideEncryption.Type() == encrypt.SSEC) ||
|
||||
(s.ServerSideEncryption.Type() == encrypt.S3 && d.ServerSideEncryption.Type() == encrypt.S3)) {
|
||||
gwMeta, err = l.getGWMetadata(ctx, srcBucket, getDareMetaPath(srcObject))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
header := make(http.Header)
|
||||
if d.ServerSideEncryption != nil {
|
||||
d.ServerSideEncryption.Marshal(header)
|
||||
}
|
||||
for k, v := range header {
|
||||
srcInfo.UserDefined[k] = v[0]
|
||||
}
|
||||
gwMeta.Meta = srcInfo.UserDefined
|
||||
if err = l.writeGWMetadata(ctx, dstBucket, getDareMetaPath(dstObject), gwMeta, minio.ObjectOptions{}); err != nil {
|
||||
return objInfo, minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
return gwMeta.ToObjectInfo(dstBucket, dstObject), nil
|
||||
}
|
||||
}
|
||||
return l.PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, srcInfo.UserDefined, d)
|
||||
}
|
||||
|
||||
// DeleteObject deletes a blob in bucket
|
||||
// For custom gateway encrypted large objects, cleans up encrypted content and metadata files
|
||||
// from the backend.
|
||||
func (l *s3EncObjects) DeleteObject(ctx context.Context, bucket string, object string) error {
|
||||
|
||||
// Get dare meta json
|
||||
if _, err := l.getGWMetadata(ctx, bucket, getDareMetaPath(object)); err != nil {
|
||||
return l.s3Objects.DeleteObject(ctx, bucket, object)
|
||||
}
|
||||
// delete encrypted object
|
||||
l.s3Objects.DeleteObject(ctx, bucket, getGWContentPath(object))
|
||||
return l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
|
||||
}
|
||||
|
||||
// ListMultipartUploads lists all multipart uploads.
|
||||
func (l *s3EncObjects) ListMultipartUploads(ctx context.Context, bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi minio.ListMultipartsInfo, e error) {
|
||||
|
||||
lmi, e = l.s3Objects.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if e != nil {
|
||||
return
|
||||
}
|
||||
lmi.KeyMarker = strings.TrimSuffix(lmi.KeyMarker, getGWContentPath("/"))
|
||||
lmi.NextKeyMarker = strings.TrimSuffix(lmi.NextKeyMarker, getGWContentPath("/"))
|
||||
for i := range lmi.Uploads {
|
||||
lmi.Uploads[i].Object = strings.TrimSuffix(lmi.Uploads[i].Object, getGWContentPath("/"))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// NewMultipartUpload uploads object in multiple parts
|
||||
func (l *s3EncObjects) NewMultipartUpload(ctx context.Context, bucket string, object string, metadata map[string]string, o minio.ObjectOptions) (uploadID string, err error) {
|
||||
var opts minio.ObjectOptions
|
||||
if o.ServerSideEncryption != nil &&
|
||||
((minio.GlobalGatewaySSE.SSEC() && o.ServerSideEncryption.Type() == encrypt.SSEC) ||
|
||||
(minio.GlobalGatewaySSE.SSES3() && o.ServerSideEncryption.Type() == encrypt.S3)) {
|
||||
opts = o
|
||||
}
|
||||
if o.ServerSideEncryption == nil {
|
||||
return l.s3Objects.NewMultipartUpload(ctx, bucket, object, metadata, opts)
|
||||
}
|
||||
uploadID, err = l.s3Objects.NewMultipartUpload(ctx, bucket, getGWContentPath(object), map[string]string{}, opts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// Create uploadID and write a temporary dare.meta object under object/uploadID prefix
|
||||
gwmeta := newGWMetaV1()
|
||||
gwmeta.Meta = metadata
|
||||
gwmeta.Stat.ModTime = time.Now().UTC()
|
||||
err = l.writeGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID), gwmeta, minio.ObjectOptions{})
|
||||
if err != nil {
|
||||
return uploadID, minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data,
|
||||
func (l *s3EncObjects) PutObject(ctx context.Context, bucket string, object string, data *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
var s3Opts minio.ObjectOptions
|
||||
if opts.ServerSideEncryption != nil &&
|
||||
((minio.GlobalGatewaySSE.SSEC() && opts.ServerSideEncryption.Type() == encrypt.SSEC) ||
|
||||
(minio.GlobalGatewaySSE.SSES3() && opts.ServerSideEncryption.Type() == encrypt.S3)) {
|
||||
s3Opts = opts
|
||||
}
|
||||
if opts.ServerSideEncryption == nil {
|
||||
defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
|
||||
defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
|
||||
return l.s3Objects.PutObject(ctx, bucket, object, data, metadata, s3Opts)
|
||||
}
|
||||
|
||||
oi, err := l.s3Objects.PutObject(ctx, bucket, getGWContentPath(object), data, map[string]string{}, s3Opts)
|
||||
if err != nil {
|
||||
return objInfo, minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
|
||||
gwMeta := newGWMetaV1()
|
||||
gwMeta.Meta = make(map[string]string)
|
||||
for k, v := range oi.UserDefined {
|
||||
gwMeta.Meta[k] = v
|
||||
}
|
||||
for k, v := range metadata {
|
||||
gwMeta.Meta[k] = v
|
||||
}
|
||||
encMD5 := data.MD5CurrentHexString()
|
||||
|
||||
gwMeta.ETag = encMD5
|
||||
gwMeta.Stat.Size = oi.Size
|
||||
gwMeta.Stat.ModTime = time.Now().UTC()
|
||||
if err = l.writeGWMetadata(ctx, bucket, getDareMetaPath(object), gwMeta, minio.ObjectOptions{}); err != nil {
|
||||
return objInfo, minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
objInfo = gwMeta.ToObjectInfo(bucket, object)
|
||||
// delete any unencrypted content of the same name created previously
|
||||
l.s3Objects.DeleteObject(ctx, bucket, object)
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (l *s3EncObjects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
|
||||
|
||||
if opts.ServerSideEncryption == nil {
|
||||
return l.s3Objects.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
}
|
||||
|
||||
var s3Opts minio.ObjectOptions
|
||||
// for sse-s3 encryption options should not be passed to backend
|
||||
if opts.ServerSideEncryption != nil && opts.ServerSideEncryption.Type() == encrypt.SSEC && minio.GlobalGatewaySSE.SSEC() {
|
||||
s3Opts = opts
|
||||
}
|
||||
|
||||
uploadPath := getTmpGWMetaPath(object, uploadID)
|
||||
tmpDareMeta := path.Join(uploadPath, gwdareMetaJSON)
|
||||
_, err := l.s3Objects.GetObjectInfo(ctx, bucket, tmpDareMeta, minio.ObjectOptions{})
|
||||
if err != nil {
|
||||
return pi, minio.InvalidUploadID{UploadID: uploadID}
|
||||
}
|
||||
|
||||
pi, e = l.s3Objects.PutObjectPart(ctx, bucket, getGWContentPath(object), uploadID, partID, data, s3Opts)
|
||||
if e != nil {
|
||||
return
|
||||
}
|
||||
gwMeta := newGWMetaV1()
|
||||
gwMeta.Parts = make([]minio.ObjectPartInfo, 1)
|
||||
// Add incoming part.
|
||||
gwMeta.Parts[0] = minio.ObjectPartInfo{
|
||||
Number: partID,
|
||||
ETag: pi.ETag,
|
||||
Size: pi.Size,
|
||||
Name: strconv.Itoa(partID),
|
||||
}
|
||||
gwMeta.ETag = data.MD5CurrentHexString() // encrypted ETag
|
||||
gwMeta.Stat.Size = pi.Size
|
||||
gwMeta.Stat.ModTime = pi.LastModified
|
||||
|
||||
if err = l.writeGWMetadata(ctx, bucket, getPartMetaPath(object, uploadID, partID), gwMeta, minio.ObjectOptions{}); err != nil {
|
||||
return pi, minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
return minio.PartInfo{
|
||||
Size: gwMeta.Stat.Size,
|
||||
ETag: minio.CanonicalizeETag(gwMeta.ETag),
|
||||
LastModified: gwMeta.Stat.ModTime,
|
||||
PartNumber: partID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// CopyObjectPart creates a part in a multipart upload by copying
|
||||
// existing object or a part of it.
|
||||
func (l *s3EncObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
|
||||
partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
|
||||
return l.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, srcInfo.PutObjReader, dstOpts)
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (l *s3EncObjects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, e error) {
|
||||
// We do not store parts uploaded so far in the dare.meta. Only CompleteMultipartUpload finalizes the parts under upload prefix.Otherwise,
|
||||
// there could be situations of dare.meta getting corrupted by competing upload parts.
|
||||
dm, err := l.getGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID))
|
||||
if err != nil {
|
||||
return l.s3Objects.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
|
||||
}
|
||||
|
||||
lpi, err = l.s3Objects.ListObjectParts(ctx, bucket, getGWContentPath(object), uploadID, partNumberMarker, maxParts, opts)
|
||||
if err != nil {
|
||||
return lpi, err
|
||||
}
|
||||
for i, part := range lpi.Parts {
|
||||
partMeta, err := l.getGWMetadata(ctx, bucket, getPartMetaPath(object, uploadID, part.PartNumber))
|
||||
if err != nil || len(partMeta.Parts) == 0 {
|
||||
return lpi, minio.InvalidPart{}
|
||||
}
|
||||
lpi.Parts[i].ETag = partMeta.ETag
|
||||
}
|
||||
lpi.UserDefined = dm.Meta
|
||||
lpi.Object = object
|
||||
return lpi, nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload aborts a ongoing multipart upload
|
||||
func (l *s3EncObjects) AbortMultipartUpload(ctx context.Context, bucket string, object string, uploadID string) error {
|
||||
if _, err := l.getGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID)); err != nil {
|
||||
return l.s3Objects.AbortMultipartUpload(ctx, bucket, object, uploadID)
|
||||
}
|
||||
|
||||
if err := l.s3Objects.AbortMultipartUpload(ctx, bucket, getGWContentPath(object), uploadID); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
uploadPrefix := getTmpGWMetaPath(object, uploadID)
|
||||
var continuationToken, startAfter, delimiter string
|
||||
for {
|
||||
loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, uploadPrefix, continuationToken, delimiter, 1000, false, startAfter)
|
||||
if err != nil {
|
||||
return minio.InvalidUploadID{UploadID: uploadID}
|
||||
}
|
||||
for _, obj := range loi.Objects {
|
||||
if err := l.s3Objects.DeleteObject(ctx, bucket, obj.Name); err != nil {
|
||||
return minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
startAfter = obj.Name
|
||||
}
|
||||
continuationToken = loi.NextContinuationToken
|
||||
if !loi.IsTruncated {
|
||||
break
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (l *s3EncObjects) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, e error) {
|
||||
|
||||
tmpMeta, err := l.getGWMetadata(ctx, bucket, getTmpDareMetaPath(object, uploadID))
|
||||
if err != nil {
|
||||
oi, e = l.s3Objects.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
if e == nil {
|
||||
// delete any encrypted version of object that might exist
|
||||
defer l.deleteGWMetadata(ctx, bucket, getDareMetaPath(object))
|
||||
defer l.DeleteObject(ctx, bucket, getGWContentPath(object))
|
||||
}
|
||||
return oi, e
|
||||
}
|
||||
gwMeta := newGWMetaV1()
|
||||
gwMeta.Meta = make(map[string]string)
|
||||
for k, v := range tmpMeta.Meta {
|
||||
gwMeta.Meta[k] = v
|
||||
}
|
||||
// Allocate parts similar to incoming slice.
|
||||
gwMeta.Parts = make([]minio.ObjectPartInfo, len(uploadedParts))
|
||||
|
||||
bkUploadedParts := make([]minio.CompletePart, len(uploadedParts))
|
||||
// Calculate full object size.
|
||||
var objectSize int64
|
||||
|
||||
// Validate each part and then commit to disk.
|
||||
for i, part := range uploadedParts {
|
||||
partMeta, err := l.getGWMetadata(ctx, bucket, getPartMetaPath(object, uploadID, part.PartNumber))
|
||||
if err != nil || len(partMeta.Parts) == 0 {
|
||||
return oi, minio.InvalidPart{}
|
||||
}
|
||||
bkUploadedParts[i] = minio.CompletePart{PartNumber: part.PartNumber, ETag: partMeta.Parts[0].ETag}
|
||||
gwMeta.Parts[i] = partMeta.Parts[0]
|
||||
objectSize += partMeta.Parts[0].Size
|
||||
}
|
||||
oi, e = l.s3Objects.CompleteMultipartUpload(ctx, bucket, getGWContentPath(object), uploadID, bkUploadedParts, opts)
|
||||
if e != nil {
|
||||
return oi, e
|
||||
}
|
||||
|
||||
//delete any unencrypted version of object that might be on the backend
|
||||
defer l.s3Objects.DeleteObject(ctx, bucket, object)
|
||||
|
||||
// Save the final object size and modtime.
|
||||
gwMeta.Stat.Size = objectSize
|
||||
gwMeta.Stat.ModTime = time.Now().UTC()
|
||||
gwMeta.ETag = oi.ETag
|
||||
|
||||
if err = l.writeGWMetadata(ctx, bucket, getDareMetaPath(object), gwMeta, minio.ObjectOptions{}); err != nil {
|
||||
return oi, minio.ErrorRespToObjectError(err)
|
||||
}
|
||||
// Clean up any uploaded parts that are not being committed by this CompleteMultipart operation
|
||||
var continuationToken, startAfter, delimiter string
|
||||
uploadPrefix := getTmpGWMetaPath(object, uploadID)
|
||||
done := false
|
||||
for {
|
||||
loi, lerr := l.s3Objects.ListObjectsV2(ctx, bucket, uploadPrefix, continuationToken, delimiter, 1000, false, startAfter)
|
||||
if lerr != nil {
|
||||
done = true
|
||||
break
|
||||
}
|
||||
for _, obj := range loi.Objects {
|
||||
if !strings.HasPrefix(obj.Name, uploadPrefix) {
|
||||
done = true
|
||||
break
|
||||
}
|
||||
startAfter = obj.Name
|
||||
l.s3Objects.DeleteObject(ctx, bucket, obj.Name)
|
||||
}
|
||||
continuationToken = loi.NextContinuationToken
|
||||
if !loi.IsTruncated || done {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return gwMeta.ToObjectInfo(bucket, object), nil
|
||||
}
|
||||
|
||||
// getTmpGWMetaPath returns the prefix under which uploads in progress are stored on backend
|
||||
func getTmpGWMetaPath(object, uploadID string) string {
|
||||
return path.Join(object, defaultMinioGWPrefix, uploadID)
|
||||
}
|
||||
|
||||
// getGWMetaPath returns the prefix under which custom object metadata and object are stored on backend after upload completes
|
||||
func getGWMetaPath(object string) string {
|
||||
return path.Join(object, defaultMinioGWPrefix)
|
||||
}
|
||||
|
||||
// getGWContentPath returns the prefix under which custom object is stored on backend after upload completes
|
||||
func getGWContentPath(object string) string {
|
||||
return path.Join(object, defaultMinioGWPrefix, defaultGWContentFileName)
|
||||
}
|
||||
|
||||
// Clean-up the stale incomplete encrypted multipart uploads. Should be run in a Go routine.
|
||||
func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration, doneCh chan struct{}) {
|
||||
ticker := time.NewTicker(cleanupInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-doneCh:
|
||||
return
|
||||
case <-ticker.C:
|
||||
l.cleanupStaleEncMultipartUploadsOnGW(ctx, expiry)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cleanupStaleMultipartUploads removes old custom encryption multipart uploads on backend
|
||||
func (l *s3EncObjects) cleanupStaleEncMultipartUploadsOnGW(ctx context.Context, expiry time.Duration) {
|
||||
for {
|
||||
buckets, err := l.s3Objects.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, b := range buckets {
|
||||
expParts := l.getStalePartsForBucket(ctx, b.Name, expiry)
|
||||
for k := range expParts {
|
||||
l.s3Objects.DeleteObject(ctx, b.Name, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (l *s3EncObjects) getStalePartsForBucket(ctx context.Context, bucket string, expiry time.Duration) (expParts map[string]string) {
|
||||
var prefix, continuationToken, delimiter, startAfter string
|
||||
expParts = make(map[string]string)
|
||||
now := time.Now()
|
||||
for {
|
||||
loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, false, startAfter)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, obj := range loi.Objects {
|
||||
startAfter = obj.Name
|
||||
if !strings.Contains(obj.Name, defaultMinioGWPrefix) {
|
||||
continue
|
||||
}
|
||||
|
||||
if isGWObject(obj.Name) {
|
||||
continue
|
||||
}
|
||||
|
||||
// delete temporary part.meta or dare.meta files for incomplete uploads that are past expiry
|
||||
if (strings.HasSuffix(obj.Name, gwpartMetaJSON) || strings.HasSuffix(obj.Name, gwdareMetaJSON)) &&
|
||||
now.Sub(obj.ModTime) > expiry {
|
||||
expParts[obj.Name] = ""
|
||||
}
|
||||
}
|
||||
continuationToken = loi.NextContinuationToken
|
||||
if !loi.IsTruncated {
|
||||
break
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (l *s3EncObjects) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
var prefix, continuationToken, delimiter, startAfter string
|
||||
expParts := make(map[string]string)
|
||||
|
||||
for {
|
||||
loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, false, startAfter)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
for _, obj := range loi.Objects {
|
||||
startAfter = obj.Name
|
||||
if !strings.Contains(obj.Name, defaultMinioGWPrefix) {
|
||||
return minio.BucketNotEmpty{}
|
||||
}
|
||||
if isGWObject(obj.Name) {
|
||||
return minio.BucketNotEmpty{}
|
||||
}
|
||||
// delete temporary part.meta or dare.meta files for incomplete uploads
|
||||
if strings.HasSuffix(obj.Name, gwpartMetaJSON) || strings.HasSuffix(obj.Name, gwdareMetaJSON) {
|
||||
expParts[obj.Name] = ""
|
||||
}
|
||||
}
|
||||
continuationToken = loi.NextContinuationToken
|
||||
if !loi.IsTruncated {
|
||||
break
|
||||
}
|
||||
}
|
||||
for k := range expParts {
|
||||
l.s3Objects.DeleteObject(ctx, bucket, k)
|
||||
}
|
||||
err := l.Client.RemoveBucket(bucket)
|
||||
if err != nil {
|
||||
return minio.ErrorRespToObjectError(err, bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package s3
|
||||
|
||||
import minio "github.com/minio/minio/cmd"
|
||||
|
||||
// List of header keys to be filtered, usually
|
||||
// from all S3 API http responses.
|
||||
var defaultFilterKeys = []string{
|
||||
"Connection",
|
||||
"Transfer-Encoding",
|
||||
"Accept-Ranges",
|
||||
"Date",
|
||||
"Server",
|
||||
"Vary",
|
||||
"x-amz-bucket-region",
|
||||
"x-amz-request-id",
|
||||
"x-amz-id-2",
|
||||
"Content-Security-Policy",
|
||||
"X-Xss-Protection",
|
||||
|
||||
// Add new headers to be ignored.
|
||||
}
|
||||
|
||||
// FromGatewayObjectPart converts ObjectInfo for custom part stored as object to PartInfo
|
||||
func FromGatewayObjectPart(partID int, oi minio.ObjectInfo) (pi minio.PartInfo) {
|
||||
return minio.PartInfo{
|
||||
Size: oi.Size,
|
||||
ETag: minio.CanonicalizeETag(oi.ETag),
|
||||
LastModified: oi.ModTime,
|
||||
PartNumber: partID,
|
||||
}
|
||||
}
|
||||
@@ -28,13 +28,13 @@ import (
|
||||
"github.com/minio/cli"
|
||||
miniogo "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/credentials"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
|
||||
"github.com/minio/minio-go/pkg/encrypt"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/policy"
|
||||
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -71,6 +71,9 @@ ENVIRONMENT VARIABLES:
|
||||
MINIO_CACHE_EXPIRY: Cache expiry duration in days.
|
||||
MINIO_CACHE_MAXUSE: Maximum permitted usage of the cache in percentage (0-100).
|
||||
|
||||
LOGGER:
|
||||
MINIO_LOGGER_HTTP_ENDPOINT: HTTP endpoint URL to log all incoming requests.
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
@@ -82,7 +85,13 @@ EXAMPLES:
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ {{.HelpName}} https://play.minio.io:9000
|
||||
|
||||
3. Start minio gateway server for AWS S3 backend with edge caching enabled.
|
||||
3. Start minio gateway server for AWS S3 backend logging all requests to http endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ export MINIO_LOGGER_HTTP_ENDPOINT="http://localhost:8000/"
|
||||
$ {{.HelpName}} https://play.minio.io:9000
|
||||
|
||||
4. Start minio gateway server for AWS S3 backend with edge caching enabled.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
@@ -103,7 +112,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: s3Backend,
|
||||
Usage: "Amazon Simple Storage Service (S3).",
|
||||
Usage: "Amazon Simple Storage Service (S3)",
|
||||
Action: s3GatewayMain,
|
||||
CustomHelpTemplate: s3GatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -194,6 +203,9 @@ func newS3(url string) (*miniogo.Core, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set custom transport
|
||||
clnt.SetCustomTransport(minio.NewCustomHTTPTransport())
|
||||
|
||||
probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
|
||||
// Check if the provided keys are valid.
|
||||
if _, err = clnt.BucketExists(probeBucketName); err != nil {
|
||||
@@ -212,9 +224,20 @@ func (g *S3) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &s3Objects{
|
||||
s := s3Objects{
|
||||
Client: clnt,
|
||||
}, nil
|
||||
}
|
||||
// Enables single encyption of KMS is configured.
|
||||
if minio.GlobalKMS != nil {
|
||||
encS := s3EncObjects{s}
|
||||
|
||||
// Start stale enc multipart uploads cleanup routine.
|
||||
go encS.cleanupStaleEncMultipartUploads(context.Background(),
|
||||
minio.GlobalMultipartCleanupInterval, minio.GlobalMultipartExpiry, minio.GlobalServiceDoneCh)
|
||||
|
||||
return &encS, nil
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
// Production - s3 gateway is production ready.
|
||||
@@ -319,6 +342,7 @@ func (l *s3Objects) ListObjects(ctx context.Context, bucket string, prefix strin
|
||||
|
||||
// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3Objects) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi minio.ListObjectsV2Info, e error) {
|
||||
|
||||
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys, startAfter)
|
||||
if err != nil {
|
||||
return loi, minio.ErrorRespToObjectError(err, bucket)
|
||||
@@ -332,13 +356,13 @@ func (l *s3Objects) GetObjectNInfo(ctx context.Context, bucket, object string, r
|
||||
var objInfo minio.ObjectInfo
|
||||
objInfo, err = l.GetObjectInfo(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
|
||||
var startOffset, length int64
|
||||
startOffset, length, err = rs.GetOffsetLength(objInfo.Size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
|
||||
pr, pw := io.Pipe()
|
||||
@@ -368,7 +392,6 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
|
||||
|
||||
if startOffset >= 0 && length >= 0 {
|
||||
if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ErrorRespToObjectError(err, bucket, key)
|
||||
}
|
||||
}
|
||||
@@ -377,9 +400,7 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
|
||||
return minio.ErrorRespToObjectError(err, bucket, key)
|
||||
}
|
||||
defer object.Close()
|
||||
|
||||
if _, err := io.Copy(writer, object); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ErrorRespToObjectError(err, bucket, key)
|
||||
}
|
||||
return nil
|
||||
@@ -387,7 +408,11 @@ func (l *s3Objects) GetObject(ctx context.Context, bucket string, key string, st
|
||||
|
||||
// GetObjectInfo reads object info and replies back ObjectInfo
|
||||
func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{miniogo.GetObjectOptions{ServerSideEncryption: opts.ServerSideEncryption}})
|
||||
oi, err := l.Client.StatObject(bucket, object, miniogo.StatObjectOptions{
|
||||
GetObjectOptions: miniogo.GetObjectOptions{
|
||||
ServerSideEncryption: opts.ServerSideEncryption,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
return minio.ObjectInfo{}, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
@@ -396,11 +421,15 @@ func (l *s3Objects) GetObjectInfo(ctx context.Context, bucket string, object str
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data,
|
||||
func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
func (l *s3Objects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
data := r.Reader
|
||||
oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), minio.ToMinioClientMetadata(metadata), opts.ServerSideEncryption)
|
||||
if err != nil {
|
||||
return objInfo, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
// On success, populate the key & metadata so they are present in the notification
|
||||
oi.Key = object
|
||||
oi.Metadata = minio.ToMinioClientObjectInfoMetadata(metadata)
|
||||
|
||||
return minio.FromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
@@ -413,6 +442,18 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
|
||||
// So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API.
|
||||
srcInfo.UserDefined["x-amz-metadata-directive"] = "REPLACE"
|
||||
srcInfo.UserDefined["x-amz-copy-source-if-match"] = srcInfo.ETag
|
||||
header := make(http.Header)
|
||||
if srcOpts.ServerSideEncryption != nil {
|
||||
encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
|
||||
}
|
||||
|
||||
if dstOpts.ServerSideEncryption != nil {
|
||||
dstOpts.ServerSideEncryption.Marshal(header)
|
||||
}
|
||||
for k, v := range header {
|
||||
srcInfo.UserDefined[k] = v[0]
|
||||
}
|
||||
|
||||
if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined); err != nil {
|
||||
return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
|
||||
}
|
||||
@@ -451,7 +492,8 @@ func (l *s3Objects) NewMultipartUpload(ctx context.Context, bucket string, objec
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, data *hash.Reader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
|
||||
func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object string, uploadID string, partID int, r *minio.PutObjReader, opts minio.ObjectOptions) (pi minio.PartInfo, e error) {
|
||||
data := r.Reader
|
||||
info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5Base64String(), data.SHA256HexString(), opts.ServerSideEncryption)
|
||||
if err != nil {
|
||||
return pi, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
@@ -464,10 +506,21 @@ func (l *s3Objects) PutObjectPart(ctx context.Context, bucket string, object str
|
||||
// existing object or a part of it.
|
||||
func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject, uploadID string,
|
||||
partID int, startOffset, length int64, srcInfo minio.ObjectInfo, srcOpts, dstOpts minio.ObjectOptions) (p minio.PartInfo, err error) {
|
||||
|
||||
srcInfo.UserDefined = map[string]string{
|
||||
"x-amz-copy-source-if-match": srcInfo.ETag,
|
||||
}
|
||||
header := make(http.Header)
|
||||
if srcOpts.ServerSideEncryption != nil {
|
||||
encrypt.SSECopy(srcOpts.ServerSideEncryption).Marshal(header)
|
||||
}
|
||||
|
||||
if dstOpts.ServerSideEncryption != nil {
|
||||
dstOpts.ServerSideEncryption.Marshal(header)
|
||||
}
|
||||
for k, v := range header {
|
||||
srcInfo.UserDefined[k] = v[0]
|
||||
}
|
||||
|
||||
completePart, err := l.Client.CopyObjectPart(srcBucket, srcObject, destBucket, destObject,
|
||||
uploadID, partID, startOffset, length, srcInfo.UserDefined)
|
||||
if err != nil {
|
||||
@@ -479,10 +532,10 @@ func (l *s3Objects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, de
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi minio.ListPartsInfo, e error) {
|
||||
func (l *s3Objects) ListObjectParts(ctx context.Context, bucket string, object string, uploadID string, partNumberMarker int, maxParts int, opts minio.ObjectOptions) (lpi minio.ListPartsInfo, e error) {
|
||||
result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
return lpi, err
|
||||
return lpi, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
|
||||
return minio.FromMinioClientListPartsInfo(result), nil
|
||||
@@ -495,13 +548,13 @@ func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart) (oi minio.ObjectInfo, e error) {
|
||||
err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
|
||||
func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, e error) {
|
||||
etag, err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
|
||||
if err != nil {
|
||||
return oi, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
|
||||
return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
|
||||
return minio.ObjectInfo{Bucket: bucket, Name: object, ETag: etag}, nil
|
||||
}
|
||||
|
||||
// SetBucketPolicy sets policy on bucket
|
||||
@@ -543,3 +596,8 @@ func (l *s3Objects) DeleteBucketPolicy(ctx context.Context, bucket string) error
|
||||
func (l *s3Objects) IsCompressionSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
|
||||
func (l *s3Objects) IsEncryptionSupported() bool {
|
||||
return minio.GlobalKMS != nil || len(minio.GlobalGatewaySSE) > 0
|
||||
}
|
||||
|
||||
@@ -50,6 +50,10 @@ func TestS3ToObjectError(t *testing.T) {
|
||||
inputErr: errResponse("InvalidBucketName"),
|
||||
expectedErr: minio.BucketNameInvalid{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("InvalidPart"),
|
||||
expectedErr: minio.InvalidPart{},
|
||||
},
|
||||
{
|
||||
inputErr: errResponse("NoSuchBucketPolicy"),
|
||||
expectedErr: minio.BucketPolicyNotFound{},
|
||||
@@ -106,8 +110,8 @@ func TestS3ToObjectError(t *testing.T) {
|
||||
// Special test case for error that is not of type
|
||||
// miniogo.ErrorResponse
|
||||
{
|
||||
inputErr: fmt.Errorf("not a minio.ErrorResponse"),
|
||||
expectedErr: fmt.Errorf("not a minio.ErrorResponse"),
|
||||
inputErr: fmt.Errorf("not a ErrorResponse"),
|
||||
expectedErr: fmt.Errorf("not a ErrorResponse"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@@ -38,7 +38,6 @@ import (
|
||||
minio "github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/sha256-simd"
|
||||
)
|
||||
|
||||
@@ -98,7 +97,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: siaBackend,
|
||||
Usage: "Sia Decentralized Cloud.",
|
||||
Usage: "Sia Decentralized Cloud",
|
||||
Action: siaGatewayMain,
|
||||
CustomHelpTemplate: siaGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -554,7 +553,8 @@ func (s *siaObjects) GetObjectInfo(ctx context.Context, bucket string, object st
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data,
|
||||
func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, data *hash.Reader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
func (s *siaObjects) PutObject(ctx context.Context, bucket string, object string, r *minio.PutObjReader, metadata map[string]string, opts minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
|
||||
data := r.Reader
|
||||
srcFile := path.Join(s.TempDir, minio.MustGetUUID())
|
||||
writer, err := os.Create(srcFile)
|
||||
if err != nil {
|
||||
|
||||
+94
-35
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -28,6 +27,7 @@ import (
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
@@ -90,7 +90,7 @@ func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
|
||||
// of the user-defined metadata to 2 KB.
|
||||
func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if isHTTPHeaderSizeTooLarge(r.Header) {
|
||||
writeErrorResponse(w, ErrMetadataTooLarge, r.URL)
|
||||
writeErrorResponse(w, ErrMetadataTooLarge, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
h.Handler.ServeHTTP(w, r)
|
||||
@@ -133,7 +133,7 @@ func filterReservedMetadata(h http.Handler) http.Handler {
|
||||
// would be treated as metadata.
|
||||
func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if containsReservedMetadata(r.Header) {
|
||||
writeErrorResponse(w, ErrUnsupportedMetadata, r.URL)
|
||||
writeErrorResponse(w, ErrUnsupportedMetadata, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
h.Handler.ServeHTTP(w, r)
|
||||
@@ -227,7 +227,8 @@ func guessIsRPCReq(req *http.Request) bool {
|
||||
if req == nil {
|
||||
return false
|
||||
}
|
||||
return req.Method == http.MethodPost
|
||||
return req.Method == http.MethodPost &&
|
||||
strings.HasPrefix(req.URL.Path, minioReservedBucketPath+"/")
|
||||
}
|
||||
|
||||
func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -299,7 +300,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
|
||||
// buckets
|
||||
bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
|
||||
if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
|
||||
writeErrorResponse(w, ErrAllAccessDisabled, r.URL)
|
||||
writeErrorResponse(w, ErrAllAccessDisabled, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -362,14 +363,14 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// All our internal APIs are sensitive towards Date
|
||||
// header, for all requests where Date header is not
|
||||
// present we will reject such clients.
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
|
||||
// or in the future, reject request otherwise.
|
||||
curTime := UTCNow()
|
||||
if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
|
||||
writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL)
|
||||
writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -484,22 +485,17 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// If bucketName is present and not objectName check for bucket level resource queries.
|
||||
if bucketName != "" && objectName == "" {
|
||||
if ignoreNotImplementedBucketResources(r) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
// If bucketName and objectName are present check for its resource queries.
|
||||
if bucketName != "" && objectName != "" {
|
||||
if ignoreNotImplementedObjectResources(r) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
// A put method on path "/" doesn't make sense, ignore it.
|
||||
if r.Method == http.MethodPut && r.URL.Path == "/" {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Serve HTTP.
|
||||
h.handler.ServeHTTP(w, r)
|
||||
@@ -600,14 +596,14 @@ func hasBadPathComponent(path string) bool {
|
||||
func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Check for bad components in URL path.
|
||||
if hasBadPathComponent(r.URL.Path) {
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Check for bad components in URL query values.
|
||||
for _, vv := range r.URL.Query() {
|
||||
for _, v := range vv {
|
||||
if hasBadPathComponent(v) {
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -624,10 +620,53 @@ type bucketForwardingHandler struct {
|
||||
}
|
||||
|
||||
func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if globalDNSConfig == nil || globalDomainName == "" || guessIsBrowserReq(r) || guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || guessIsRPCReq(r) {
|
||||
if globalDNSConfig == nil || globalDomainName == "" ||
|
||||
guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
|
||||
guessIsRPCReq(r) || isAdminReq(r) {
|
||||
f.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// For browser requests, when federation is setup we need to
|
||||
// specifically handle download and upload for browser requests.
|
||||
if guessIsBrowserReq(r) && globalDNSConfig != nil && globalDomainName != "" {
|
||||
var bucket, _ string
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
if getRequestAuthType(r) == authTypeJWT {
|
||||
bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/upload"))
|
||||
}
|
||||
case http.MethodGet:
|
||||
if t := r.URL.Query().Get("token"); t != "" {
|
||||
bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/download"))
|
||||
} else if getRequestAuthType(r) != authTypeJWT && !strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
|
||||
bucket, _ = urlPath2BucketObjectName(r.URL.Path)
|
||||
}
|
||||
}
|
||||
if bucket != "" {
|
||||
sr, err := globalDNSConfig.Get(bucket)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
|
||||
} else {
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
r.URL.Scheme = "http"
|
||||
if globalIsSSL {
|
||||
r.URL.Scheme = "https"
|
||||
}
|
||||
r.URL.Host = getHostFromSrv(sr)
|
||||
f.fwd.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
f.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
bucket, object := urlPath2BucketObjectName(r.URL.Path)
|
||||
// ListBucket requests should be handled at current endpoint as
|
||||
// all buckets data can be fetched from here.
|
||||
@@ -652,19 +691,18 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
|
||||
sr, err := globalDNSConfig.Get(bucket)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL)
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
|
||||
} else {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
host, port := getRandomHostPort(sr)
|
||||
r.URL.Scheme = "http"
|
||||
if globalIsSSL {
|
||||
r.URL.Scheme = "https"
|
||||
}
|
||||
r.URL.Host = fmt.Sprintf("%s:%d", host, port)
|
||||
r.URL.Host = getHostFromSrv(sr)
|
||||
f.fwd.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
@@ -715,31 +753,34 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// potential pileup on the server.
|
||||
if err := l.Wait(ctx); err != nil {
|
||||
// Send an S3 compatible error, SlowDown.
|
||||
writeErrorResponse(w, ErrSlowDown, r.URL)
|
||||
writeErrorResponse(w, ErrSlowDown, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
l.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// requestIDHeaderHandler sets x-amz-request-id header.
|
||||
// Previously, this value was set right before a response
|
||||
// was sent to the client.So, logger and Error response XML
|
||||
// were not using this value.
|
||||
// This is set here so that this header can be logged as
|
||||
// part of the log entry and Error response XML.
|
||||
type requestIDHeaderHandler struct {
|
||||
// customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header.
|
||||
// Previously, this value was set right before a response was sent to
|
||||
// the client. So, logger and Error response XML were not using this
|
||||
// value. This is set here so that this header can be logged as
|
||||
// part of the log entry, Error response XML and auditing.
|
||||
type customHeaderHandler struct {
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func addrequestIDHeader(h http.Handler) http.Handler {
|
||||
return requestIDHeaderHandler{handler: h}
|
||||
func addCustomHeaders(h http.Handler) http.Handler {
|
||||
return customHeaderHandler{handler: h}
|
||||
}
|
||||
|
||||
func (s requestIDHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Set unique request ID for each response.
|
||||
func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Set custom headers such as x-amz-request-id and x-minio-deployment-id
|
||||
// for each request.
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
|
||||
s.handler.ServeHTTP(w, r)
|
||||
if globalDeploymentID != "" {
|
||||
w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
|
||||
}
|
||||
s.handler.ServeHTTP(logger.NewResponseWriter(w), r)
|
||||
}
|
||||
|
||||
type securityHeaderHandler struct {
|
||||
@@ -766,10 +807,28 @@ type criticalErrorHandler struct{ handler http.Handler }
|
||||
func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err == logger.ErrCritical { // handle
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
} else if err != nil {
|
||||
panic(err) // forward other panic calls
|
||||
}
|
||||
}()
|
||||
h.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func setSSETLSHandler(h http.Handler) http.Handler { return sseTLSHandler{h} }
|
||||
|
||||
// sseTLSHandler enforces certain rules for SSE requests which are made / must be made over TLS.
|
||||
type sseTLSHandler struct{ handler http.Handler }
|
||||
|
||||
func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Deny SSE-C requests if not made over TLS
|
||||
if !globalIsSSL && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
|
||||
if r.Method == http.MethodHead {
|
||||
writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
|
||||
} else {
|
||||
writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
h.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
@@ -76,9 +78,16 @@ func TestGuessIsRPC(t *testing.T) {
|
||||
if guessIsRPCReq(nil) {
|
||||
t.Fatal("Unexpected return for nil request")
|
||||
}
|
||||
|
||||
u, err := url.Parse("http://localhost:9000/minio/lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := &http.Request{
|
||||
Proto: "HTTP/1.0",
|
||||
Method: http.MethodPost,
|
||||
URL: u,
|
||||
}
|
||||
if !guessIsRPCReq(r) {
|
||||
t.Fatal("Test shouldn't fail for a possible net/rpc request.")
|
||||
@@ -181,3 +190,41 @@ func TestContainsReservedMetadata(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var sseTLSHandlerTests = []struct {
|
||||
URL *url.URL
|
||||
Header http.Header
|
||||
IsTLS, ShouldFail bool
|
||||
}{
|
||||
{URL: &url.URL{}, Header: http.Header{}, IsTLS: false, ShouldFail: false}, // 0
|
||||
{URL: &url.URL{}, Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: false, ShouldFail: true}, // 1
|
||||
{URL: &url.URL{}, Header: http.Header{crypto.SSECAlgorithm: []string{"AES256"}}, IsTLS: true, ShouldFail: false}, // 2
|
||||
{URL: &url.URL{}, Header: http.Header{crypto.SSECKey: []string{""}}, IsTLS: true, ShouldFail: false}, // 3
|
||||
{URL: &url.URL{}, Header: http.Header{crypto.SSECopyAlgorithm: []string{""}}, IsTLS: false, ShouldFail: true}, // 4
|
||||
}
|
||||
|
||||
func TestSSETLSHandler(t *testing.T) {
|
||||
defer func(isSSL bool) { globalIsSSL = isSSL }(globalIsSSL) // reset globalIsSSL after test
|
||||
|
||||
var okHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
for i, test := range sseTLSHandlerTests {
|
||||
globalIsSSL = test.IsTLS
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := new(http.Request)
|
||||
r.Header = test.Header
|
||||
r.URL = test.URL
|
||||
|
||||
h := setSSETLSHandler(okHandler)
|
||||
h.ServeHTTP(w, r)
|
||||
|
||||
switch {
|
||||
case test.ShouldFail && w.Code == http.StatusOK:
|
||||
t.Errorf("Test %d: should fail but status code is HTTP %d", i, w.Code)
|
||||
case !test.ShouldFail && w.Code != http.StatusOK:
|
||||
t.Errorf("Test %d: should not fail but status code is HTTP %d and not 200 OK", i, w.Code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+39
-9
@@ -34,12 +34,16 @@ import (
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/certs"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
"github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/iam/validator"
|
||||
)
|
||||
|
||||
// minio configuration related constants.
|
||||
const (
|
||||
globalMinioCertExpireWarnDays = time.Hour * 24 * 30 // 30 days.
|
||||
|
||||
globalMinioDefaultPort = "9000"
|
||||
|
||||
globalMinioDefaultRegion = ""
|
||||
// This is a sha256 output of ``arn:aws:iam::minio:user/admin``,
|
||||
// this is kept in present form to be compatible with S3 owner ID
|
||||
@@ -75,17 +79,26 @@ const (
|
||||
// date and server date during signature verification.
|
||||
globalMaxSkewTime = 15 * time.Minute // 15 minutes skew allowed.
|
||||
|
||||
// Expiry duration after which the multipart uploads are deemed stale.
|
||||
globalMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
|
||||
// Cleanup interval when the stale multipart cleanup is initiated.
|
||||
globalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
|
||||
// GlobalMultipartExpiry - Expiry duration after which the multipart uploads are deemed stale.
|
||||
GlobalMultipartExpiry = time.Hour * 24 * 14 // 2 weeks.
|
||||
// GlobalMultipartCleanupInterval - Cleanup interval when the stale multipart cleanup is initiated.
|
||||
GlobalMultipartCleanupInterval = time.Hour * 24 // 24 hrs.
|
||||
|
||||
// Refresh interval to update in-memory bucket policy cache.
|
||||
globalRefreshBucketPolicyInterval = 5 * time.Minute
|
||||
// Refresh interval to update in-memory iam config cache.
|
||||
globalRefreshIAMInterval = 5 * time.Minute
|
||||
|
||||
// Limit of location constraint XML for unauthenticted PUT bucket operations.
|
||||
maxLocationConstraintSize = 3 * humanize.MiByte
|
||||
)
|
||||
|
||||
var globalCLIContext = struct {
|
||||
JSON, Quiet bool
|
||||
Anonymous bool
|
||||
Addr string
|
||||
}{}
|
||||
|
||||
var (
|
||||
// Indicates the total number of erasure coded sets configured.
|
||||
globalXLSetCount int
|
||||
@@ -123,7 +136,7 @@ var (
|
||||
// Minio local server address (in `host:port` format)
|
||||
globalMinioAddr = ""
|
||||
// Minio default port, can be changed through command line.
|
||||
globalMinioPort = "9000"
|
||||
globalMinioPort = globalMinioDefaultPort
|
||||
// Holds the host that was passed using --address
|
||||
globalMinioHost = ""
|
||||
|
||||
@@ -132,6 +145,7 @@ var (
|
||||
|
||||
globalNotificationSys *NotificationSys
|
||||
globalPolicySys *PolicySys
|
||||
globalIAMSys *IAMSys
|
||||
|
||||
// CA root certificates, a nil value means system certs pool will be used
|
||||
globalRootCAs *x509.CertPool
|
||||
@@ -223,10 +237,14 @@ var (
|
||||
|
||||
// KMS key id
|
||||
globalKMSKeyID string
|
||||
// Allocated KMS
|
||||
globalKMS crypto.KMS
|
||||
// KMS config
|
||||
globalKMSConfig crypto.KMSConfig
|
||||
|
||||
// GlobalKMS initialized KMS configuration
|
||||
GlobalKMS crypto.KMS
|
||||
|
||||
// Auto-Encryption, if enabled, turns any non-SSE-C request
|
||||
// into an SSE-S3 request. If enabled a valid, non-empty KMS
|
||||
// configuration must be present.
|
||||
globalAutoEncryption bool
|
||||
|
||||
// Is compression include extensions/content-types set.
|
||||
globalIsEnvCompression bool
|
||||
@@ -244,6 +262,18 @@ var (
|
||||
// Some standard content-types which we strictly dis-allow for compression.
|
||||
standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"}
|
||||
|
||||
// Authorization validators list.
|
||||
globalIAMValidators *validator.Validators
|
||||
|
||||
// OPA policy system.
|
||||
globalPolicyOPA *iampolicy.Opa
|
||||
|
||||
// Deployment ID - unique per deployment
|
||||
globalDeploymentID string
|
||||
|
||||
// GlobalGatewaySSE sse options
|
||||
GlobalGatewaySSE gatewaySSE
|
||||
|
||||
// Add new variable global values here.
|
||||
)
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user