mirror of
https://github.com/pgsty/minio.git
synced 2026-11-17 04:49:33 +03:00
Compare commits
82 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 236796ebd6 | |||
| 4e4f855b30 | |||
| 2db22deb93 | |||
| fb8d0d7cf7 | |||
| a536cf5dc0 | |||
| b9b68e9331 | |||
| 632022971b | |||
| def04f01cf | |||
| bc67410548 | |||
| 7881791a91 | |||
| d2f8f8c7ee | |||
| 8c32311b80 | |||
| 9bb88e610e | |||
| d1e41695fe | |||
| 2aeb3fbe86 | |||
| 99b843a64e | |||
| 4f31a9a33b | |||
| 65ddff8899 | |||
| e7c902bbbc | |||
| 5a5895203b | |||
| 7da0336ac8 | |||
| 3be616de3f | |||
| c5bf22fd90 | |||
| 7c9f934875 | |||
| b6f9b24b30 | |||
| 13cb814a0e | |||
| d264d2c899 | |||
| bebaff269c | |||
| 52b159b1db | |||
| 324834e4da | |||
| c2ed1347d9 | |||
| 50f6f9fe58 | |||
| 48cb0ea34b | |||
| 6f7c99a333 | |||
| 3498f5b0ec | |||
| 21d8c0fd13 | |||
| 79b9a9ce46 | |||
| b9b353db4b | |||
| 11a9b317a3 | |||
| 9af7d627ac | |||
| 76d9d54603 | |||
| 4c7c571875 | |||
| 313ba74b09 | |||
| 3e124315c8 | |||
| 78a0fd951e | |||
| 40852801ea | |||
| f6980c4630 | |||
| 8fcc787cba | |||
| 4e6d3c093f | |||
| 61145361fd | |||
| 950b4ad9af | |||
| 6add646130 | |||
| 20e61fb362 | |||
| 18ced1102c | |||
| d6af3c1237 | |||
| 5549a44566 | |||
| e7af31c2ff | |||
| e7971b1d55 | |||
| 26120d7838 | |||
| bef7c01c58 | |||
| 6a8ccc5925 | |||
| d85199e9de | |||
| 8608a84c23 | |||
| b7226f4c82 | |||
| f930ffe9e2 | |||
| b50a245208 | |||
| 45bb11e020 | |||
| b65cf281fd | |||
| f781548b0c | |||
| 25ee8e74f7 | |||
| c975d2cc7e | |||
| ea66528739 | |||
| e1164103d4 | |||
| 83fe70f710 | |||
| 12a6523fb2 | |||
| dba61867e8 | |||
| dd092f6c2b | |||
| 2a810c7da2 | |||
| dd8c2aa5c6 | |||
| 9e3fce441e | |||
| d4265f9a13 | |||
| 2fc024e880 |
@@ -34,6 +34,11 @@ matrix:
|
||||
- for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
|
||||
- bash buildscripts/go-coverage.sh
|
||||
|
||||
before_script:
|
||||
# Add an IPv6 config - see the corresponding Travis issue
|
||||
# https://github.com/travis-ci/travis-ci/issues/8361
|
||||
- if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
|
||||
|
||||
before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
|
||||
|
||||
|
||||
+15
-11
@@ -1,28 +1,32 @@
|
||||
FROM golang:1.10.1-alpine3.7
|
||||
FROM golang:1.10.4-alpine3.7
|
||||
|
||||
LABEL maintainer="Minio Inc <dev@minio.io>"
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
|
||||
RUN \
|
||||
apk add --no-cache git && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)"
|
||||
|
||||
FROM alpine:3.7
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
EXPOSE 9000
|
||||
|
||||
COPY --from=0 /go/bin/minio /usr/bin/minio
|
||||
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates 'curl>7.61.0' && \
|
||||
apk add --no-cache --virtual .build-deps git && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
|
||||
rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
|
||||
@@ -68,6 +68,36 @@ function start_minio_erasure_sets()
|
||||
echo "$minio_pid"
|
||||
}
|
||||
|
||||
function start_minio_dist_erasure_sets_ipv6()
|
||||
{
|
||||
declare -a minio_pids
|
||||
export MINIO_ACCESS_KEY=$ACCESS_KEY
|
||||
export MINIO_SECRET_KEY=$SECRET_KEY
|
||||
"${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9000.log" 2>&1 &
|
||||
minio_pids[0]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9001.log" 2>&1 &
|
||||
minio_pids[1]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9002" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9002.log" 2>&1 &
|
||||
minio_pids[2]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9003" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9003.log" 2>&1 &
|
||||
minio_pids[3]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9004" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9004.log" 2>&1 &
|
||||
minio_pids[4]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9005" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9005.log" 2>&1 &
|
||||
minio_pids[5]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9006" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9006.log" 2>&1 &
|
||||
minio_pids[6]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9007" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9007.log" 2>&1 &
|
||||
minio_pids[7]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9008" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9008.log" 2>&1 &
|
||||
minio_pids[8]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9009" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9009.log" 2>&1 &
|
||||
minio_pids[9]=$!
|
||||
|
||||
sleep 35
|
||||
echo "${minio_pids[@]}"
|
||||
}
|
||||
|
||||
function start_minio_dist_erasure_sets()
|
||||
{
|
||||
declare -a minio_pids
|
||||
@@ -161,6 +191,34 @@ function run_test_erasure_sets() {
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function run_test_dist_erasure_sets_ipv6()
|
||||
{
|
||||
minio_pids=( $(start_minio_dist_erasure_sets_ipv6) )
|
||||
|
||||
export SERVER_ENDPOINT="[::1]:9000"
|
||||
|
||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||
rv=$?
|
||||
|
||||
for pid in "${minio_pids[@]}"; do
|
||||
kill "$pid"
|
||||
done
|
||||
sleep 3
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
for i in $(seq 0 9); do
|
||||
echo "server$i log:"
|
||||
cat "$WORK_DIR/dist-minio-v6-900$i.log"
|
||||
done
|
||||
fi
|
||||
|
||||
for i in $(seq 0 9); do
|
||||
rm -f "$WORK_DIR/dist-minio-v6-900$i.log"
|
||||
done
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function run_test_dist_erasure_sets()
|
||||
{
|
||||
minio_pids=( $(start_minio_dist_erasure_sets) )
|
||||
@@ -237,6 +295,7 @@ function run_test_gateway_s3()
|
||||
{
|
||||
minio_pid="$(start_minio_gateway_s3)"
|
||||
|
||||
export SERVER_ENDPOINT="127.0.0.1:9000"
|
||||
export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||
@@ -279,6 +338,7 @@ function __init__()
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
|
||||
chmod a+x "$FUNCTIONAL_TESTS"
|
||||
}
|
||||
|
||||
@@ -319,6 +379,13 @@ function main()
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Distributed Erasure setup as sets with ipv6"
|
||||
if ! run_test_dist_erasure_sets_ipv6; then
|
||||
echo "FAILED"
|
||||
rm -fr "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Gateway S3 setup"
|
||||
if ! run_test_gateway_s3; then
|
||||
echo "FAILED"
|
||||
|
||||
+10
-10
@@ -57,28 +57,28 @@ type accessControlPolicy struct {
|
||||
func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketACL")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketACL")
|
||||
defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Allow getBucketACL if policy action is set, since this is a dummy call
|
||||
// we are simply re-purposing the bucketPolicyAction.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
Permission: "FULL_CONTROL",
|
||||
})
|
||||
if err := xml.NewEncoder(w).Encode(acl); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetObjectACL")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetObjectACL")
|
||||
defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -114,21 +114,21 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Allow getObjectACL if policy action is set, since this is a dummy call
|
||||
// we are simply re-purposing the bucketPolicyAction.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Before proceeding validate if object exists.
|
||||
_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
|
||||
Permission: "FULL_CONTROL",
|
||||
})
|
||||
if err := xml.NewEncoder(w).Encode(acl); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+43
-3
@@ -578,7 +578,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
StartTime: nh.startTime,
|
||||
})
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Client token not specified but a heal sequence exists on a path,
|
||||
@@ -859,7 +859,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
// Custom IAM policies not allowed for admin user.
|
||||
if accessKey == globalServerConfig.GetCredential().AccessKey {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -867,6 +867,14 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddUser - PUT /minio/admin/v1/add-user?accessKey=<access_key>
|
||||
@@ -898,7 +906,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Custom IAM policies not allowed for admin user.
|
||||
if accessKey == globalServerConfig.GetCredential().AccessKey {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -927,6 +935,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
|
||||
@@ -992,6 +1008,14 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddCannedPolicy - PUT /minio/admin/v1/add-canned-policy?name=<policy_name>
|
||||
@@ -1049,6 +1073,14 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetUserPolicy - PUT /minio/admin/v1/set-user-policy?accessKey=<access_key>&name=<policy_name>
|
||||
@@ -1088,6 +1120,14 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
|
||||
if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetConfigHandler - PUT /minio/admin/v1/config
|
||||
|
||||
@@ -38,7 +38,7 @@ import (
|
||||
|
||||
var (
|
||||
configJSON = []byte(`{
|
||||
"version": "32",
|
||||
"version": "33",
|
||||
"credential": {
|
||||
"accessKey": "minio",
|
||||
"secretKey": "minio123"
|
||||
@@ -152,7 +152,6 @@ var (
|
||||
"streaming": {
|
||||
"enable": false,
|
||||
"clusterID": "",
|
||||
"clientID": "",
|
||||
"async": false,
|
||||
"maxPubAcksInflight": 0
|
||||
}
|
||||
@@ -278,7 +277,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := mux.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
registerAdminRouter(adminRouter, true)
|
||||
|
||||
return &adminXLTestBed{
|
||||
xlDirs: xlDirs,
|
||||
|
||||
@@ -641,7 +641,12 @@ func (h *healSequence) healDiskFormat() error {
|
||||
// Healing succeeded notify the peers to reload format and re-initialize disks.
|
||||
// We will not notify peers only if healing succeeded.
|
||||
if err == nil {
|
||||
peersReInitFormat(globalAdminPeers, h.settings.DryRun)
|
||||
for host, rerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
|
||||
if rerr != nil {
|
||||
logger.GetReqInfo(h.ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(h.ctx, rerr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Push format heal result
|
||||
|
||||
+32
-30
@@ -31,7 +31,7 @@ type adminAPIHandlers struct {
|
||||
}
|
||||
|
||||
// registerAdminRouter - Add handler functions for each service REST API routes.
|
||||
func registerAdminRouter(router *mux.Router) {
|
||||
func registerAdminRouter(router *mux.Router, enableIAM bool) {
|
||||
|
||||
adminAPI := adminAPIHandlers{}
|
||||
// Admin router
|
||||
@@ -69,42 +69,44 @@ func registerAdminRouter(router *mux.Router) {
|
||||
|
||||
/// Config operations
|
||||
|
||||
// Update credentials
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
|
||||
// Get config
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
|
||||
// Set config
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
|
||||
if enableIAM {
|
||||
// Update credentials
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
|
||||
// Get config
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
|
||||
// Set config
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
|
||||
|
||||
// Get config keys/values
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
|
||||
// Set config keys/values
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
|
||||
// Get config keys/values
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
|
||||
// Set config keys/values
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
|
||||
|
||||
// -- IAM APIs --
|
||||
// -- IAM APIs --
|
||||
|
||||
// Add policy IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
|
||||
// Add policy IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
|
||||
|
||||
// Add user IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
|
||||
// Add user IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
|
||||
|
||||
// Remove policy IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
|
||||
// Remove policy IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
|
||||
|
||||
// Remove user IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
// Remove user IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
// List users
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
|
||||
// List users
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
|
||||
|
||||
// List policies
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
|
||||
// List policies
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
|
||||
}
|
||||
|
||||
// If none of the routes match.
|
||||
adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
|
||||
// If none of the routes match, return error.
|
||||
adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
|
||||
}
|
||||
|
||||
+5
-39
@@ -45,29 +45,12 @@ func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error)
|
||||
return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
|
||||
}
|
||||
|
||||
// ReInitFormat - re-initialize disk format, remotely.
|
||||
func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
|
||||
args := ReInitFormatArgs{DryRun: dryRun}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
|
||||
}
|
||||
|
||||
// ServerInfo - returns the server info of the server to which the RPC call is made.
|
||||
func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
|
||||
err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
|
||||
return sid, err
|
||||
}
|
||||
|
||||
// GetConfig - returns config.json of the remote server.
|
||||
func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
|
||||
args := AuthArgs{}
|
||||
var reply []byte
|
||||
|
||||
err := rpcClient.Call(adminServiceName+".GetConfig", &args, &reply)
|
||||
return reply, err
|
||||
}
|
||||
|
||||
// StartProfiling - starts profiling in the remote server.
|
||||
func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
|
||||
args := StartProfilingArgs{Profiler: profiler}
|
||||
@@ -125,9 +108,7 @@ func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
|
||||
// commands like service stop and service restart.
|
||||
type adminCmdRunner interface {
|
||||
SignalService(s serviceSignal) error
|
||||
ReInitFormat(dryRun bool) error
|
||||
ServerInfo() (ServerInfoData, error)
|
||||
GetConfig() ([]byte, error)
|
||||
StartProfiling(string) error
|
||||
DownloadProfilingData() ([]byte, error)
|
||||
}
|
||||
@@ -149,6 +130,11 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
// Use first IPv4 instead of loopback address.
|
||||
localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
|
||||
}
|
||||
if strings.HasPrefix(localAddr, "[::1]:") {
|
||||
// Use first IPv4 instead of loopback address.
|
||||
localAddr = net.JoinHostPort(localIP6.ToSlice()[0], globalMinioPort)
|
||||
}
|
||||
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
addr: localAddr,
|
||||
cmdRunner: localAdminClient{},
|
||||
@@ -169,26 +155,6 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
return adminPeerList
|
||||
}
|
||||
|
||||
// peersReInitFormat - reinitialize remote object layers to new format.
|
||||
func peersReInitFormat(peers adminPeers, dryRun bool) error {
|
||||
errs := make([]error, len(peers))
|
||||
|
||||
// Send ReInitFormat RPC call to all nodes.
|
||||
// for local adminPeer this is a no-op.
|
||||
wg := sync.WaitGroup{}
|
||||
for i, peer := range peers {
|
||||
wg.Add(1)
|
||||
go func(idx int, peer adminPeer) {
|
||||
defer wg.Done()
|
||||
if !peer.isLocal {
|
||||
errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
|
||||
}
|
||||
}(i, peer)
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Initialize global adminPeer collection.
|
||||
func initGlobalAdminPeers(endpoints EndpointList) {
|
||||
globalAdminPeers = makeAdminPeers(endpoints)
|
||||
|
||||
@@ -68,23 +68,6 @@ func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[
|
||||
return
|
||||
}
|
||||
|
||||
// GetConfig - returns the config.json of this server.
|
||||
func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
|
||||
*reply, err = receiver.local.GetConfig()
|
||||
return err
|
||||
}
|
||||
|
||||
// ReInitFormatArgs - provides dry-run information to re-initialize format.json
|
||||
type ReInitFormatArgs struct {
|
||||
AuthArgs
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// ReInitFormat - re-init 'format.json'
|
||||
func (receiver *adminRPCReceiver) ReInitFormat(args *ReInitFormatArgs, reply *VoidReply) error {
|
||||
return receiver.local.ReInitFormat(args.DryRun)
|
||||
}
|
||||
|
||||
// NewAdminRPCServer - returns new admin RPC server.
|
||||
func NewAdminRPCServer() (*xrpc.Server, error) {
|
||||
rpcServer := xrpc.NewServer()
|
||||
|
||||
+1
-79
@@ -40,11 +40,8 @@ import (
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalServiceSignalCh := globalServiceSignalCh
|
||||
defer func(sigChan chan serviceSignal) { globalServiceSignalCh = sigChan }(globalServiceSignalCh)
|
||||
globalServiceSignalCh = make(chan serviceSignal, 10)
|
||||
defer func() {
|
||||
globalServiceSignalCh = tmpGlobalServiceSignalCh
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
signal serviceSignal
|
||||
@@ -66,34 +63,6 @@ func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalObjectAPI := globalObjectAPI
|
||||
defer func() {
|
||||
globalObjectAPI = tmpGlobalObjectAPI
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
objectAPI ObjectLayer
|
||||
dryRun bool
|
||||
expectErr bool
|
||||
}{
|
||||
{&DummyObjectLayer{}, true, false},
|
||||
{&DummyObjectLayer{}, false, false},
|
||||
{nil, true, true},
|
||||
{nil, false, true},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
globalObjectAPI = testCase.objectAPI
|
||||
err := client.ReInitFormat(testCase.dryRun)
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalBootTime := globalBootTime
|
||||
tmpGlobalObjectAPI := globalObjectAPI
|
||||
@@ -140,33 +109,6 @@ func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerGetConfig(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalServerConfig := globalServerConfig
|
||||
defer func() {
|
||||
globalServerConfig = tmpGlobalServerConfig
|
||||
}()
|
||||
|
||||
config := newServerConfig()
|
||||
|
||||
testCases := []struct {
|
||||
config *serverConfig
|
||||
expectErr bool
|
||||
}{
|
||||
{globalServerConfig, false},
|
||||
{config, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
globalServerConfig = testCase.config
|
||||
_, err := client.GetConfig()
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
|
||||
rpcServer, err := NewAdminRPCServer()
|
||||
if err != nil {
|
||||
@@ -208,16 +150,6 @@ func TestAdminRPCClientSignalService(t *testing.T) {
|
||||
testAdminCmdRunnerSignalService(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientReInitFormat(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
defer func() {
|
||||
globalServerConfig = prevGlobalServerConfig
|
||||
}()
|
||||
|
||||
testAdminCmdRunnerReInitFormat(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientServerInfo(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
@@ -227,13 +159,3 @@ func TestAdminRPCClientServerInfo(t *testing.T) {
|
||||
|
||||
testAdminCmdRunnerServerInfo(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientGetConfig(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
defer func() {
|
||||
globalServerConfig = prevGlobalServerConfig
|
||||
}()
|
||||
|
||||
testAdminCmdRunnerGetConfig(t, rpcClient)
|
||||
}
|
||||
|
||||
@@ -87,6 +87,7 @@ const (
|
||||
ErrNoSuchBucketPolicy
|
||||
ErrNoSuchKey
|
||||
ErrNoSuchUpload
|
||||
ErrNoSuchVersion
|
||||
ErrNotImplemented
|
||||
ErrPreconditionFailed
|
||||
ErrRequestTimeTooSkewed
|
||||
@@ -447,6 +448,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNoSuchVersion: {
|
||||
Code: "NoSuchVersion",
|
||||
Description: "Indicates that the version ID specified in the request does not match an existing version.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNotImplemented: {
|
||||
Code: "NotImplemented",
|
||||
Description: "A header you provided implies functionality that is not implemented",
|
||||
|
||||
+10
-1
@@ -568,13 +568,22 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
|
||||
}
|
||||
|
||||
// writeErrorRespone writes error headers
|
||||
func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
|
||||
func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL, browser bool) {
|
||||
switch errorCode {
|
||||
case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set("Retry-After", "120")
|
||||
case ErrAccessDenied:
|
||||
// The request is from browser and also if browser
|
||||
// is enabled we need to redirect.
|
||||
if browser && globalIsBrowserEnabled {
|
||||
w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
|
||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
apiError := getAPIError(errorCode)
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
|
||||
|
||||
+58
-41
@@ -19,10 +19,10 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@@ -155,49 +155,66 @@ func getSessionToken(r *http.Request) (token string) {
|
||||
return r.URL.Query().Get("X-Amz-Security-Token")
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client and validate the token.
|
||||
func getClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
|
||||
stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
|
||||
if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
|
||||
}
|
||||
if err := jwtToken.Claims.Valid(); err != nil {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
if claims, ok := jwtToken.Claims.(jwtgo.MapClaims); ok {
|
||||
if _, ok = claims["accessKey"].(string); !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
// JWT token for x-amz-security-token is signed with admin
|
||||
// secret key, temporary credentials become invalid if
|
||||
// server admin credentials change. This is done to ensure
|
||||
// that clients cannot decode the token using the temp
|
||||
// secret keys and generate an entirely new claim by essentially
|
||||
// hijacking the policies. We need to make sure that this is
|
||||
// based an admin credential such that token cannot be decoded
|
||||
// on the client side and is treated like an opaque value.
|
||||
return []byte(globalServerConfig.GetCredential().SecretKey), nil
|
||||
}
|
||||
return nil, errAuthentication
|
||||
}
|
||||
// Fetch claims in the security token returned by the client, doesn't return
|
||||
// errors - upon errors the returned claims map will be empty.
|
||||
func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
|
||||
claims, _ := getClaimsFromToken(r)
|
||||
return claims
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client.
|
||||
func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
|
||||
claims := make(map[string]interface{})
|
||||
token := getSessionToken(r)
|
||||
if token == "" {
|
||||
return nil, ErrNone
|
||||
return claims, nil
|
||||
}
|
||||
stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
|
||||
// JWT token for x-amz-security-token is signed with admin
|
||||
// secret key, temporary credentials become invalid if
|
||||
// server admin credentials change. This is done to ensure
|
||||
// that clients cannot decode the token using the temp
|
||||
// secret keys and generate an entirely new claim by essentially
|
||||
// hijacking the policies. We need to make sure that this is
|
||||
// based an admin credential such that token cannot be decoded
|
||||
// on the client side and is treated like an opaque value.
|
||||
return []byte(globalServerConfig.GetCredential().SecretKey), nil
|
||||
}
|
||||
p := &jwtgo.Parser{
|
||||
ValidMethods: []string{
|
||||
jwtgo.SigningMethodHS256.Alg(),
|
||||
jwtgo.SigningMethodHS512.Alg(),
|
||||
},
|
||||
}
|
||||
jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !jtoken.Valid {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
v, ok := claims["accessKey"]
|
||||
if !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
if _, ok = v.(string); !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client and validate the token.
|
||||
func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
|
||||
token := getSessionToken(r)
|
||||
if token != "" && cred.AccessKey == "" {
|
||||
return nil, ErrNoAccessKey
|
||||
}
|
||||
if token != cred.SessionToken {
|
||||
if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
|
||||
return nil, ErrInvalidToken
|
||||
}
|
||||
p := &jwtgo.Parser{}
|
||||
jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
|
||||
claims, err := getClaimsFromToken(r)
|
||||
if err != nil {
|
||||
return nil, toAPIErrorCode(context.Background(), errAuthentication)
|
||||
}
|
||||
if !jtoken.Valid {
|
||||
return nil, toAPIErrorCode(context.Background(), errAuthentication)
|
||||
return nil, toAPIErrorCode(context.Background(), err)
|
||||
}
|
||||
return claims, ErrNone
|
||||
}
|
||||
@@ -233,6 +250,11 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
||||
return s3Err
|
||||
}
|
||||
|
||||
claims, s3Err := checkClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// LocationConstraint is valid only for CreateBucketAction.
|
||||
var locationConstraint string
|
||||
if action == policy.CreateBucketAction {
|
||||
@@ -256,11 +278,6 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
|
||||
}
|
||||
|
||||
claims, s3Err := getClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
if cred.AccessKey == "" {
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
@@ -399,7 +416,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
a.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
|
||||
writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
|
||||
// isPutAllowed - check if PUT operation is allowed on the resource, this
|
||||
@@ -421,7 +438,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
|
||||
return s3Err
|
||||
}
|
||||
|
||||
claims, s3Err := getClaimsFromToken(r, cred)
|
||||
claims, s3Err := checkClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
@@ -59,19 +59,19 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
|
||||
func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectsV2")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListObjectsV2")
|
||||
defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -81,14 +81,14 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
|
||||
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponse(w, errCode, r.URL)
|
||||
writeErrorResponse(w, errCode, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the query params before beginning to serve the request.
|
||||
// fetch-owner is not validated since it is a boolean
|
||||
if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listObjectsV2 := objectAPI.ListObjectsV2
|
||||
@@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// Read the decompressed size from the meta.json.
|
||||
actualSize = listObjectsV2Info.Objects[i].GetActualSize()
|
||||
if actualSize < 0 {
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Set the info.Size to the actualSize.
|
||||
@@ -119,7 +119,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
|
||||
listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -141,37 +141,37 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectsV1")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListObjectsV1")
|
||||
defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Extract all the litsObjectsV1 query params to their native values.
|
||||
prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
|
||||
// does not throw an error.
|
||||
if maxKeys < 0 {
|
||||
writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidMaxKeys, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
} // Validate all the query params before beginning to serve the request.
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listObjects := objectAPI.ListObjects
|
||||
@@ -183,7 +183,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -193,7 +193,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
// Read the decompressed size from the meta.json.
|
||||
actualSize = listObjectsInfo.Objects[i].GetActualSize()
|
||||
if actualSize < 0 {
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Set the info.Size to the actualSize.
|
||||
@@ -202,7 +202,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
|
||||
listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
+93
-79
@@ -89,19 +89,19 @@ func initFederatorBackend(objLayer ObjectLayer) {
|
||||
func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketLocation")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketLocation")
|
||||
defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
getBucketInfo = api.CacheAPI().GetBucketInfo
|
||||
}
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -139,42 +139,42 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListMultipartUploads")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListMultipartUploads")
|
||||
defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if maxUploads < 0 {
|
||||
writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidMaxUploads, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if keyMarker != "" {
|
||||
// Marker not common with prefix is not implemented.
|
||||
if !hasPrefix(keyMarker, prefix) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// generate response
|
||||
@@ -192,11 +192,11 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListBuckets")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListBuckets")
|
||||
defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listBuckets := objectAPI.ListBuckets
|
||||
@@ -206,7 +206,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// If etcd, dns federation configured list buckets from etcd.
|
||||
@@ -214,7 +214,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
if globalDNSConfig != nil {
|
||||
dnsBuckets, err := globalDNSConfig.List()
|
||||
if err != nil && err != dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
bucketSet := set.NewStringSet()
|
||||
@@ -233,7 +233,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
var err error
|
||||
bucketsInfo, err = listBuckets(ctx)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -250,14 +250,14 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteMultipleObjects")
|
||||
|
||||
defer logger.AuditLog(w, r, "DeleteMultipleObjects")
|
||||
defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -266,7 +266,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// In the event access is denied, a 200 response should still be returned
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if s3Error != ErrAccessDenied {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -274,14 +274,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Content-Length is required and should be non-zero
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Content-Md5 is requied should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if _, ok := r.Header["Content-Md5"]; !ok {
|
||||
writeErrorResponse(w, ErrMissingContentMD5, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentMD5, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -297,7 +297,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Read incoming body XML bytes.
|
||||
if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -305,7 +305,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
deleteObjects := &DeleteObjectsRequest{}
|
||||
if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedXML, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedXML, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -313,7 +313,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
if globalWORMEnabled {
|
||||
// Not required to check whether given objects exist or not, because
|
||||
// DeleteMultipleObject is always successful irrespective of object existence.
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -397,11 +397,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucket")
|
||||
defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -409,21 +409,21 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Parse incoming location constraint.
|
||||
location, s3Error := parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if location sent by the client is valid, reject
|
||||
// requests which do not follow valid region requirements.
|
||||
if !isValidLocation(location) {
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -432,12 +432,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
// Proceed to creating a bucket.
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if err = globalDNSConfig.Put(bucket); err != nil {
|
||||
objectAPI.DeleteBucket(ctx, bucket)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -447,18 +447,18 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
|
||||
}
|
||||
writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
|
||||
writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -475,30 +475,37 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PostPolicyBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "PostPolicyBucket")
|
||||
defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if !objectAPI.IsEncryptionSupported() && hasServerSideEncryptionHeader(r.Header) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
bucket := mux.Vars(r)["bucket"]
|
||||
|
||||
// Require Content-Length to be set in the request
|
||||
size := r.ContentLength
|
||||
if size < 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Make sure that the URL does not contain object name.
|
||||
if bucket != filepath.Clean(resource[1:]) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -507,7 +514,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
reader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -515,7 +522,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
form, err := reader.ReadForm(maxFormMemory)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -526,13 +533,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if file is provided, error out otherwise.
|
||||
if fileBody == nil {
|
||||
writeErrorResponse(w, ErrPOSTFileRequired, r.URL)
|
||||
writeErrorResponse(w, ErrPOSTFileRequired, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -554,7 +561,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if successRedirect != "" {
|
||||
redirectURL, err = url.Parse(successRedirect)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -562,60 +569,67 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// Verify policy signature.
|
||||
apiErr := doesPolicySignatureMatch(formValues)
|
||||
if apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure formValues adhere to policy restrictions.
|
||||
if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure that the object size is within expected range, also the file size
|
||||
// should not exceed the maximum single Put size (5 GiB)
|
||||
lengthRange := postPolicyForm.Conditions.ContentLengthRange
|
||||
if lengthRange.Valid {
|
||||
if fileSize < lengthRange.Min {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL)
|
||||
// Handle policy if it is set.
|
||||
if len(policyBytes) > 0 {
|
||||
postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL)
|
||||
// Make sure formValues adhere to policy restrictions.
|
||||
if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure that the object size is within expected range, also the file size
|
||||
// should not exceed the maximum single Put size (5 GiB)
|
||||
lengthRange := postPolicyForm.Conditions.ContentLengthRange
|
||||
if lengthRange.Valid {
|
||||
if fileSize < lengthRange.Min {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract metadata to be saved from received Form.
|
||||
metadata := make(map[string]string)
|
||||
err = extractMetadataFromMap(ctx, formValues, metadata)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
rawReader := hashReader
|
||||
pReader := NewPutObjReader(rawReader, nil, nil)
|
||||
var objectEncryptionKey []byte
|
||||
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
|
||||
r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
}
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
|
||||
var reader io.Reader
|
||||
@@ -623,19 +637,19 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if crypto.SSEC.IsRequested(formValues) {
|
||||
key, err = ParseSSECustomerHeader(formValues)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
info := ObjectInfo{Size: fileSize}
|
||||
hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
|
||||
@@ -644,7 +658,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, ObjectOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -703,7 +717,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HeadBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "HeadBucket")
|
||||
defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -735,19 +749,19 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucket")
|
||||
|
||||
defer logger.AuditLog(w, r, "DeleteBucket")
|
||||
defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -757,7 +771,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
// Attempt to delete bucket.
|
||||
if err := deleteBucket(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -769,7 +783,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
if err := globalDNSConfig.Delete(bucket); err != nil {
|
||||
// Deleting DNS entry failed, attempt to create the bucket again.
|
||||
objectAPI.MakeBucketWithLocation(ctx, bucket, "")
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,30 +44,30 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
|
||||
func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketNotification")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketNotification")
|
||||
defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucketName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -76,7 +76,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
if err != nil {
|
||||
// Ignore errNoSuchNotifications to comply with AWS S3.
|
||||
if err != errNoSuchNotifications {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
|
||||
notificationBytes, err := xml.Marshal(nConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -102,16 +102,16 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketNotification")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucketNotification")
|
||||
defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objectAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -119,19 +119,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucketName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucketNotification always needs a Content-Length.
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -143,12 +143,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
apiErr = toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -164,24 +164,28 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListenBucketNotification")
|
||||
|
||||
defer logger.AuditLog(w, r, "ListenBucketNotification")
|
||||
defer logger.AuditLog(w, r, "ListenBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
// Validate if bucket exists.
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if !objAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objAPI.IsListenBucketSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -189,11 +193,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
var prefix string
|
||||
if len(values["prefix"]) > 1 {
|
||||
writeErrorResponse(w, ErrFilterNamePrefix, r.URL)
|
||||
writeErrorResponse(w, ErrFilterNamePrefix, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
if len(values["prefix"]) == 1 {
|
||||
if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -202,11 +206,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
var suffix string
|
||||
if len(values["suffix"]) > 1 {
|
||||
writeErrorResponse(w, ErrFilterNameSuffix, r.URL)
|
||||
writeErrorResponse(w, ErrFilterNameSuffix, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
if len(values["suffix"]) == 1 {
|
||||
if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -219,7 +223,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
for _, s := range values["events"] {
|
||||
eventName, err := event.ParseName(s)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -227,19 +231,19 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
host, err := xnet.ParseHost(r.RemoteAddr)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
target, err := target.NewHTTPClientTarget(*host, w)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -247,7 +251,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
|
||||
@@ -255,13 +259,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -271,7 +275,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,11 +40,11 @@ const (
|
||||
func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(w, r, "PutBucketPolicy")
|
||||
defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -52,43 +52,43 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is missing.
|
||||
// PutBucketPolicy always needs Content-Length.
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
if r.ContentLength > maxBucketPolicySize {
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL)
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Version in policy must not be empty
|
||||
if bucketPolicy.Version == "" {
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -103,11 +103,11 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(w, r, "DeleteBucketPolicy")
|
||||
defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -115,18 +115,18 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -141,11 +141,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(w, r, "GetBucketPolicy")
|
||||
defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -153,26 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Read bucket access policy.
|
||||
bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
policyData, err := json.Marshal(bucketPolicy)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+5
-1
@@ -71,9 +71,13 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
|
||||
// Get all CA file names.
|
||||
var caFiles []string
|
||||
fis, err := readDir(certsCAsDir)
|
||||
if err != nil {
|
||||
if err != nil && err != errFileNotFound {
|
||||
return nil, err
|
||||
}
|
||||
// Return success if CA's directory is missing.
|
||||
if err == errFileNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
for _, fi := range fis {
|
||||
// Skip all directories.
|
||||
if hasSuffix(fi, slashSeparator) {
|
||||
|
||||
+2
-1
@@ -223,7 +223,8 @@ func TestGetRootCAs(t *testing.T) {
|
||||
certCAsDir string
|
||||
expectedErr error
|
||||
}{
|
||||
{"nonexistent-dir", errFileNotFound},
|
||||
// ignores non-existent directories.
|
||||
{"nonexistent-dir", nil},
|
||||
// Ignores directories.
|
||||
{dir1, nil},
|
||||
// Ignore empty directory.
|
||||
|
||||
+44
-20
@@ -30,7 +30,6 @@ import (
|
||||
dns2 "github.com/miekg/dns"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/logger/target/console"
|
||||
"github.com/minio/minio/cmd/logger/target/http"
|
||||
@@ -81,6 +80,31 @@ func loadLoggers() {
|
||||
|
||||
func handleCommonCmdArgs(ctx *cli.Context) {
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if globalCLIContext.JSON {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if globalCLIContext.Quiet {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Get anonymous flag from command line argument.
|
||||
globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
|
||||
if globalCLIContext.Anonymous {
|
||||
logger.EnableAnonymous()
|
||||
}
|
||||
|
||||
// Fetch address option
|
||||
globalCLIContext.Addr = ctx.GlobalString("address")
|
||||
if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+globalMinioDefaultPort {
|
||||
globalCLIContext.Addr = ctx.String("address")
|
||||
}
|
||||
|
||||
var configDir string
|
||||
|
||||
switch {
|
||||
@@ -221,14 +245,28 @@ func handleCommonEnvVars() {
|
||||
minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if ok {
|
||||
minioEndpoints := strings.Split(minioEndpointsEnv, ",")
|
||||
globalDomainIPs = set.NewStringSet()
|
||||
for i, ip := range minioEndpoints {
|
||||
if net.ParseIP(ip) == nil {
|
||||
logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
|
||||
var domainIPs = set.NewStringSet()
|
||||
for _, endpoint := range minioEndpoints {
|
||||
if net.ParseIP(endpoint) == nil {
|
||||
// Checking if the IP is a DNS entry.
|
||||
addrs, err := net.LookupHost(endpoint)
|
||||
if err != nil {
|
||||
logger.FatalIf(err, "Unable to initialize Minio server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint)
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
domainIPs.Add(addr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
globalDomainIPs.Add(ip)
|
||||
domainIPs.Add(endpoint)
|
||||
}
|
||||
updateDomainIPs(domainIPs)
|
||||
} else {
|
||||
// Add found interfaces IP address to global domain IPS,
|
||||
// loopback addresses will be naturally dropped.
|
||||
updateDomainIPs(localIP4)
|
||||
}
|
||||
|
||||
if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
var err error
|
||||
globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
|
||||
@@ -318,20 +356,6 @@ func handleCommonEnvVars() {
|
||||
globalWORMEnabled = bool(wormFlag)
|
||||
}
|
||||
|
||||
kmsConf, err := crypto.NewVaultConfig()
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize hashicorp vault")
|
||||
}
|
||||
if kmsConf.Vault.Endpoint != "" {
|
||||
kms, err := crypto.NewVault(kmsConf)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize KMS")
|
||||
}
|
||||
globalKMS = kms
|
||||
globalKMSKeyID = kmsConf.Vault.Key.Name
|
||||
globalKMSConfig = kmsConf
|
||||
}
|
||||
|
||||
if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
|
||||
globalIsCompressionEnabled = strings.EqualFold(compress, "true")
|
||||
}
|
||||
|
||||
+10
-12
@@ -43,9 +43,9 @@ import (
|
||||
// 6. Make changes in config-current_test.go for any test change
|
||||
|
||||
// Config version
|
||||
const serverConfigVersion = "32"
|
||||
const serverConfigVersion = "33"
|
||||
|
||||
type serverConfig = serverConfigV32
|
||||
type serverConfig = serverConfigV33
|
||||
|
||||
var (
|
||||
// globalServerConfig server config.
|
||||
@@ -271,8 +271,8 @@ func (s *serverConfig) loadFromEnvs() {
|
||||
s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
|
||||
}
|
||||
|
||||
if globalKMS != nil {
|
||||
s.KMS = globalKMSConfig
|
||||
if err := Environment.LookupKMSConfig(s.KMS); err != nil {
|
||||
logger.FatalIf(err, "Unable to setup the KMS")
|
||||
}
|
||||
|
||||
if globalIsEnvCompression {
|
||||
@@ -534,12 +534,8 @@ func (s *serverConfig) loadToCachedConfigs() {
|
||||
globalCacheExpiry = cacheConf.Expiry
|
||||
globalCacheMaxUse = cacheConf.MaxUse
|
||||
}
|
||||
if globalKMS == nil {
|
||||
globalKMSConfig = s.KMS
|
||||
if kms, err := crypto.NewVault(globalKMSConfig); err == nil {
|
||||
globalKMS = kms
|
||||
globalKMSKeyID = globalKMSConfig.Vault.Key.Name
|
||||
}
|
||||
if err := Environment.LookupKMSConfig(s.KMS); err != nil {
|
||||
logger.FatalIf(err, "Unable to setup the KMS")
|
||||
}
|
||||
|
||||
if !globalIsCompressionEnabled {
|
||||
@@ -556,8 +552,10 @@ func (s *serverConfig) loadToCachedConfigs() {
|
||||
if globalPolicyOPA == nil {
|
||||
if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
|
||||
globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
|
||||
URL: s.Policy.OPA.URL,
|
||||
AuthToken: s.Policy.OPA.AuthToken,
|
||||
URL: s.Policy.OPA.URL,
|
||||
AuthToken: s.Policy.OPA.AuthToken,
|
||||
Transport: NewCustomHTTPTransport(),
|
||||
CloseRespFn: CloseResponse,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
|
||||
|
||||
// Test 12 - Test NATS
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
|
||||
// Test 13 - Test ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
|
||||
|
||||
+9
-1
@@ -74,7 +74,15 @@ func (config *ConfigDir) GetCADir() string {
|
||||
|
||||
// Create - creates configuration directory tree.
|
||||
func (config *ConfigDir) Create() error {
|
||||
return os.MkdirAll(config.GetCADir(), 0700)
|
||||
err := os.MkdirAll(config.GetCADir(), 0700)
|
||||
// It is possible in kubernetes like deployments this directory
|
||||
// is already mounted and is not writable, ignore any write errors.
|
||||
if err != nil {
|
||||
if os.IsPermission(err) {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// GetMinioConfigFile - returns absolute path of config.json file.
|
||||
|
||||
+94
-2
@@ -2413,8 +2413,64 @@ func migrateV27ToV28() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Migrates '.minio.sys/config.json' to v32.
|
||||
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
|
||||
defer func() {
|
||||
// Rename config.json to config.json.deprecated only upon
|
||||
// success of this function.
|
||||
if err == nil {
|
||||
os.Rename(getConfigFile(), getConfigFile()+".deprecated")
|
||||
}
|
||||
}()
|
||||
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
// Construct path to config.json for the given bucket.
|
||||
transactionConfigFile := configFile + ".transaction"
|
||||
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
// Verify if backend already has the file.
|
||||
if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
|
||||
return err
|
||||
} // if errConfigNotFound proceed to migrate..
|
||||
|
||||
var config = &serverConfig{}
|
||||
if _, err = Load(getConfigFile(), config); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Read from deprecate file as well if necessary.
|
||||
if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return saveServerConfig(context.Background(), objAPI, config)
|
||||
}
|
||||
|
||||
// Migrates '.minio.sys/config.json' to v33.
|
||||
func migrateMinioSysConfig(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
// Construct path to config.json for the given bucket.
|
||||
transactionConfigFile := configFile + ".transaction"
|
||||
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
if err := migrateV27ToV28MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2427,7 +2483,10 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
|
||||
if err := migrateV30ToV31MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
return migrateV31ToV32MinioSys(objAPI)
|
||||
if err := migrateV31ToV32MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
return migrateV32ToV33MinioSys(objAPI)
|
||||
}
|
||||
|
||||
func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
|
||||
@@ -2623,3 +2682,36 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "32")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := &serverConfigV33{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Version = "33"
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from 32 to 33 . %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "32", "33")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -159,8 +159,8 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test if a config migration from v2 to v30 is successfully done
|
||||
func TestServerConfigMigrateV2toV30(t *testing.T) {
|
||||
// Test if a config migration from v2 to v33 is successfully done
|
||||
func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -892,3 +892,47 @@ type serverConfigV32 struct {
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS.
|
||||
type serverConfigV33 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Worm BoolFlag `json:"worm"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// KMS configuration
|
||||
KMS crypto.KMSConfig `json:"kms"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
|
||||
// Compression configuration
|
||||
Compression compressionConfig `json:"compress"`
|
||||
|
||||
// OpenID configuration
|
||||
OpenID struct {
|
||||
// JWKS validator config.
|
||||
JWKS validator.JWKSArgs `json:"jwks"`
|
||||
} `json:"openid"`
|
||||
|
||||
// External policy enforcements.
|
||||
Policy struct {
|
||||
// OPA configuration.
|
||||
OPA iampolicy.OpaArgs `json:"opa"`
|
||||
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
@@ -156,30 +156,6 @@ func NewConfigSys() *ConfigSys {
|
||||
return &ConfigSys{}
|
||||
}
|
||||
|
||||
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
func migrateConfigToMinioSys(objAPI ObjectLayer) error {
|
||||
defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
|
||||
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
// Verify if backend already has the file.
|
||||
if err := checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
|
||||
return err
|
||||
} // if errConfigNotFound proceed to migrate..
|
||||
|
||||
var config = &serverConfig{}
|
||||
if _, err := Load(getConfigFile(), config); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Read from deprecate file as well if necessary.
|
||||
if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return saveServerConfig(context.Background(), objAPI, config)
|
||||
}
|
||||
|
||||
// Initialize and load config from remote etcd or local config directory
|
||||
func initConfig(objAPI ObjectLayer) error {
|
||||
if objAPI == nil {
|
||||
|
||||
@@ -22,16 +22,16 @@ import (
|
||||
)
|
||||
|
||||
// Writes S3 compatible copy part range error.
|
||||
func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL) {
|
||||
func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL, browser bool) {
|
||||
switch err {
|
||||
case errInvalidRange:
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRange, url)
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRange, url, browser)
|
||||
return
|
||||
case errInvalidRangeSource:
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url)
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url, browser)
|
||||
return
|
||||
default:
|
||||
writeErrorResponse(w, ErrInternalError, url)
|
||||
writeErrorResponse(w, ErrInternalError, url, browser)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,5 +16,6 @@ package crypto
|
||||
|
||||
// KMSConfig has the KMS config for hashicorp vault
|
||||
type KMSConfig struct {
|
||||
Vault VaultConfig `json:"vault"`
|
||||
AutoEncryption bool `json:"-"`
|
||||
Vault VaultConfig `json:"vault"`
|
||||
}
|
||||
|
||||
+123
-179
@@ -19,195 +19,133 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
vault "github.com/hashicorp/vault/api"
|
||||
)
|
||||
|
||||
const (
|
||||
// vaultEndpointEnv Vault endpoint environment variable
|
||||
vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
|
||||
// vaultAuthTypeEnv type of vault auth to be used
|
||||
vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
|
||||
// vaultAppRoleIDEnv Vault AppRole ID environment variable
|
||||
vaultAppRoleIDEnv = "MINIO_SSE_VAULT_APPROLE_ID"
|
||||
// vaultAppSecretIDEnv Vault AppRole Secret environment variable
|
||||
vaultAppSecretIDEnv = "MINIO_SSE_VAULT_APPROLE_SECRET"
|
||||
// vaultKeyVersionEnv Vault Key Version environment variable
|
||||
vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
|
||||
// vaultKeyNameEnv Vault Encryption Key Name environment variable
|
||||
vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
|
||||
|
||||
// vaultCAPath is the path to a directory of PEM-encoded CA
|
||||
// cert files to verify the Vault server SSL certificate.
|
||||
vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
|
||||
)
|
||||
|
||||
var (
|
||||
//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
|
||||
ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
|
||||
)
|
||||
|
||||
// VaultKey represents vault encryption key-ring.
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"` // The name of the encryption key-ring
|
||||
Version int `json:"version"` // The key version
|
||||
}
|
||||
|
||||
// VaultAuth represents vault authentication type.
|
||||
// Currently the only supported authentication type is AppRole.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"` // The authentication type
|
||||
AppRole VaultAppRole `json:"approle"` // The AppRole authentication credentials
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault AppRole authentication credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"` // The AppRole access ID
|
||||
Secret string `json:"secret"` // The AppRole secret
|
||||
}
|
||||
|
||||
// VaultConfig represents vault configuration.
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"` // The vault API endpoint as URL
|
||||
CAPath string `json:"-"` // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
|
||||
Auth VaultAuth `json:"auth"` // The vault authentication configuration
|
||||
Key VaultKey `json:"key-id"` // The named key used for key-generation / decryption.
|
||||
Namespace string `json:"-"` // The vault namespace of enterprise vault instances
|
||||
}
|
||||
|
||||
// vaultService represents a connection to a vault KMS.
|
||||
type vaultService struct {
|
||||
config *VaultConfig
|
||||
client *vault.Client
|
||||
leaseDuration time.Duration
|
||||
}
|
||||
|
||||
// return transit secret engine's path for generate data key operation
|
||||
func (v *vaultService) genDataKeyEndpoint(key string) string {
|
||||
return "/transit/datakey/plaintext/" + key
|
||||
var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
|
||||
|
||||
// empty/default vault configuration used to check whether a particular is empty.
|
||||
var emptyVaultConfig = VaultConfig{}
|
||||
|
||||
// IsEmpty returns true if the vault config struct is an
|
||||
// empty configuration.
|
||||
func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }
|
||||
|
||||
// Verify returns a nil error if the vault configuration
|
||||
// is valid. A valid configuration is either empty or
|
||||
// contains valid non-default values.
|
||||
func (v *VaultConfig) Verify() (err error) {
|
||||
if v.IsEmpty() {
|
||||
return // an empty configuration is valid
|
||||
}
|
||||
switch {
|
||||
case v.Endpoint == "":
|
||||
err = errors.New("crypto: missing hashicorp vault endpoint")
|
||||
case strings.ToLower(v.Auth.Type) != "approle":
|
||||
err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
|
||||
case v.Auth.AppRole.ID == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppRole ID")
|
||||
case v.Auth.AppRole.Secret == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppSecret ID")
|
||||
case v.Key.Name == "":
|
||||
err = errors.New("crypto: missing hashicorp vault key name")
|
||||
case v.Key.Version < 0:
|
||||
err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// return transit secret engine's path for decrypt operation
|
||||
func (v *vaultService) decryptEndpoint(key string) string {
|
||||
return "/transit/decrypt/" + key
|
||||
}
|
||||
// NewVault initializes Hashicorp Vault KMS by authenticating
|
||||
// to Vault with the credentials in config and gets a client
|
||||
// token for future api calls.
|
||||
func NewVault(config VaultConfig) (KMS, error) {
|
||||
if config.IsEmpty() {
|
||||
return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
|
||||
}
|
||||
if err := config.Verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// VaultKey represents vault encryption key-id name & version
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
}
|
||||
vaultCfg := vault.Config{Address: config.Endpoint}
|
||||
if err := vaultCfg.ConfigureTLS(&vault.TLSConfig{CAPath: config.CAPath}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := vault.NewClient(&vaultCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.Namespace != "" {
|
||||
client.SetNamespace(config.Namespace)
|
||||
}
|
||||
|
||||
// VaultAuth represents vault auth type to use. For now, AppRole is the only supported
|
||||
// auth type.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"`
|
||||
AppRole VaultAppRole `json:"approle"`
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault approle credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"`
|
||||
Secret string `json:"secret"`
|
||||
}
|
||||
|
||||
// VaultConfig holds config required to start vault service
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Auth VaultAuth `json:"auth"`
|
||||
Key VaultKey `json:"key-id"`
|
||||
}
|
||||
|
||||
// validate whether all required env variables needed to start vault service have
|
||||
// been set
|
||||
func validateVaultConfig(c *VaultConfig) error {
|
||||
if c.Endpoint == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
|
||||
payload := map[string]interface{}{
|
||||
"role_id": config.Auth.AppRole.ID,
|
||||
"secret_id": config.Auth.AppRole.Secret,
|
||||
}
|
||||
if strings.ToLower(c.Auth.Type) != "approle" {
|
||||
return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
|
||||
}
|
||||
if c.Auth.AppRole.ID == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppRole ID - %s is empty", vaultAppRoleIDEnv)
|
||||
}
|
||||
if c.Auth.AppRole.Secret == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppSecret ID - %s is empty", vaultAppSecretIDEnv)
|
||||
}
|
||||
if c.Key.Name == "" {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyNameEnv)
|
||||
}
|
||||
if c.Key.Version < 0 {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// authenticate to vault with app role id and app role secret, and get a client access token, lease duration
|
||||
func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (token string, duration int, err error) {
|
||||
data := map[string]interface{}{
|
||||
"role_id": appRoleID,
|
||||
"secret_id": appSecret,
|
||||
}
|
||||
resp, e := client.Logical().Write("auth/approle/login", data)
|
||||
if e != nil {
|
||||
return token, duration, e
|
||||
resp, err := client.Logical().Write("auth/approle/login", payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Auth == nil {
|
||||
return token, duration, ErrKMSAuthLogin
|
||||
return nil, ErrKMSAuthLogin
|
||||
}
|
||||
return resp.Auth.ClientToken, resp.Auth.LeaseDuration, nil
|
||||
|
||||
client.SetToken(resp.Auth.ClientToken)
|
||||
v := &vaultService{client: client, config: &config, leaseDuration: time.Duration(resp.Auth.LeaseDuration)}
|
||||
v.renewToken()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// NewVaultConfig sets KMSConfig from environment
|
||||
// variables and performs validations.
|
||||
func NewVaultConfig() (KMSConfig, error) {
|
||||
kc := KMSConfig{}
|
||||
endpoint := os.Getenv(vaultEndpointEnv)
|
||||
roleID := os.Getenv(vaultAppRoleIDEnv)
|
||||
roleSecret := os.Getenv(vaultAppSecretIDEnv)
|
||||
keyName := os.Getenv(vaultKeyNameEnv)
|
||||
keyVersion := 0
|
||||
authType := "approle"
|
||||
if versionStr := os.Getenv(vaultKeyVersionEnv); versionStr != "" {
|
||||
version, err := strconv.Atoi(versionStr)
|
||||
if err != nil {
|
||||
return kc, fmt.Errorf("Unable to parse %s value (`%s`)", vaultKeyVersionEnv, versionStr)
|
||||
}
|
||||
keyVersion = version
|
||||
}
|
||||
// return if none of the vault env variables are configured
|
||||
if (endpoint == "") && (roleID == "") && (roleSecret == "") && (keyName == "") && (keyVersion == 0) {
|
||||
return kc, nil
|
||||
}
|
||||
c := VaultConfig{
|
||||
Endpoint: endpoint,
|
||||
Auth: VaultAuth{
|
||||
Type: authType,
|
||||
AppRole: VaultAppRole{
|
||||
ID: roleID,
|
||||
Secret: roleSecret,
|
||||
},
|
||||
},
|
||||
Key: VaultKey{
|
||||
Version: keyVersion,
|
||||
Name: keyName,
|
||||
},
|
||||
}
|
||||
if err := validateVaultConfig(&c); err != nil {
|
||||
return kc, err
|
||||
}
|
||||
kc.Vault = c
|
||||
return kc, nil
|
||||
}
|
||||
|
||||
// NewVault initializes Hashicorp Vault KMS by
|
||||
// authenticating to Vault with the credentials in KMSConfig,
|
||||
// and gets a client token for future api calls.
|
||||
func NewVault(kmsConf KMSConfig) (KMS, error) {
|
||||
config := kmsConf.Vault
|
||||
vconfig := &vault.Config{
|
||||
Address: config.Endpoint,
|
||||
}
|
||||
if err := vconfig.ConfigureTLS(&vault.TLSConfig{
|
||||
CAPath: os.Getenv(vaultCAPath),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := vault.NewClient(vconfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accessToken, leaseDuration, err := getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// authenticate and get the access token
|
||||
c.SetToken(accessToken)
|
||||
v := vaultService{client: c, config: &config, leaseDuration: time.Duration(leaseDuration)}
|
||||
v.renewToken(c)
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (v *vaultService) renewToken(c *vault.Client) {
|
||||
// renewToken starts a new go-routine which renews
|
||||
// the vault authentication token periodically.
|
||||
func (v *vaultService) renewToken() {
|
||||
retryDelay := 1 * time.Minute
|
||||
go func() {
|
||||
for {
|
||||
s, err := c.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
if err != nil {
|
||||
time.Sleep(retryDelay)
|
||||
continue
|
||||
@@ -218,48 +156,54 @@ func (v *vaultService) renewToken(c *vault.Client) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Generates a random plain text key, sealed plain text key from
|
||||
// Vault. It returns the plaintext key and sealed plaintext key on success
|
||||
// GenerateKey returns a new plaintext key, generated by the KMS,
|
||||
// and a sealed version of this plaintext key encrypted using the
|
||||
// named key referenced by keyID. It also binds the generated key
|
||||
// cryptographically to the provided context.
|
||||
func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.genDataKeyEndpoint(keyID), payload)
|
||||
|
||||
if err1 != nil {
|
||||
return key, sealedKey, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, sealedKey, err
|
||||
}
|
||||
sealKey := s.Data["ciphertext"].(string)
|
||||
plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
|
||||
if err != nil {
|
||||
return key, sealedKey, err1
|
||||
return key, sealedKey, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
return key, []byte(sealKey), nil
|
||||
}
|
||||
|
||||
// unsealKMSKey unseals the sealedKey using the Vault master key
|
||||
// referenced by the keyID. The plain text key is returned on success.
|
||||
// UnsealKey returns the decrypted sealedKey as plaintext key.
|
||||
// Therefore it sends the sealedKey to the KMS which decrypts
|
||||
// it using the named key referenced by keyID and responses with
|
||||
// the plaintext key.
|
||||
//
|
||||
// The context must be same context as the one provided while
|
||||
// generating the plaintext key / sealedKey.
|
||||
func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"ciphertext": string(sealedKey),
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.decryptEndpoint(keyID), payload)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
base64Key := s.Data["plaintext"].(string)
|
||||
plainKey, err1 := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
plainKey, err := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
@@ -154,6 +154,9 @@ func (api *DummyObjectLayer) DeleteBucketPolicy(context.Context, string) (err er
|
||||
func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
|
||||
return
|
||||
}
|
||||
func (api *DummyObjectLayer) IsListenBucketSupported() (b bool) {
|
||||
return
|
||||
}
|
||||
|
||||
func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
|
||||
return
|
||||
|
||||
@@ -226,6 +226,17 @@ func TestGetSetIndexes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func getHexSequences(start int, number int, paddinglen int) (seq []string) {
|
||||
for i := start; i <= number; i++ {
|
||||
if paddinglen == 0 {
|
||||
seq = append(seq, fmt.Sprintf("%x", i))
|
||||
} else {
|
||||
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", paddinglen), i))
|
||||
}
|
||||
}
|
||||
return seq
|
||||
}
|
||||
|
||||
func getSequences(start int, number int, paddinglen int) (seq []string) {
|
||||
for i := start; i <= number; i++ {
|
||||
if paddinglen == 0 {
|
||||
@@ -472,6 +483,52 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
},
|
||||
true,
|
||||
},
|
||||
// IPv6 ellipses with hexadecimal expansion
|
||||
{
|
||||
"http://[2001:3984:3989::{1...a}]/disk{1...10}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
Prefix: "http://[2001:3984:3989::",
|
||||
Suffix: "]/disk",
|
||||
Seq: getHexSequences(1, 10, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// IPv6 ellipses with hexadecimal expansion with 3 position numerics.
|
||||
{
|
||||
"http://[2001:3984:3989::{001...00a}]/disk{1...10}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
Prefix: "http://[2001:3984:3989::",
|
||||
Suffix: "]/disk",
|
||||
Seq: getHexSequences(1, 10, 3),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
|
||||
+30
-23
@@ -112,6 +112,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
|
||||
}
|
||||
}
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
|
||||
if host == "" {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
|
||||
@@ -152,7 +155,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
// Only check if the arg is an ip address and ask for scheme since its absent.
|
||||
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
|
||||
// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
|
||||
if isHostIPv4(arg) {
|
||||
if isHostIP(arg) {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
|
||||
}
|
||||
u = &url.URL{Path: path.Clean(arg)}
|
||||
@@ -224,7 +227,6 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
|
||||
uniqueArgs.Add(arg)
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
@@ -341,7 +343,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
if err != nil {
|
||||
host = endpoint.Host
|
||||
}
|
||||
hostIPSet, _ := getHostIP4(host)
|
||||
hostIPSet, _ := getHostIP(host)
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return serverAddr, endpoints, setupType,
|
||||
@@ -411,12 +413,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
host = localServerAddr
|
||||
}
|
||||
|
||||
ipList, err := getHostIP4(host)
|
||||
ipList, err := getHostIP(host)
|
||||
logger.FatalIf(err, "unexpected error when resolving host '%s'", host)
|
||||
|
||||
// Filter ipList by IPs those start with '127.'.
|
||||
// Filter ipList by IPs those start with '127.' or '::1'
|
||||
loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return strings.HasPrefix(ip, "127.")
|
||||
return strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
|
||||
}, "")
|
||||
|
||||
// If loop back IP is found and ipList contains only loop back IPs, then error out.
|
||||
@@ -455,7 +457,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
updateDomainIPs(uniqueArgs)
|
||||
_, dok := os.LookupEnv("MINIO_DOMAIN")
|
||||
_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
|
||||
_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if dok && eok && !iok {
|
||||
updateDomainIPs(uniqueArgs)
|
||||
}
|
||||
|
||||
setupType = DistXLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
@@ -480,10 +487,10 @@ func GetLocalPeer(endpoints EndpointList) (localPeer string) {
|
||||
// Local peer can be empty in FS or Erasure coded mode.
|
||||
// If so, return globalMinioHost + globalMinioPort value.
|
||||
if globalMinioHost != "" {
|
||||
return globalMinioHost + ":" + globalMinioPort
|
||||
return net.JoinHostPort(globalMinioHost, globalMinioPort)
|
||||
}
|
||||
|
||||
return "127.0.0.1:" + globalMinioPort
|
||||
return net.JoinHostPort("127.0.0.1", globalMinioPort)
|
||||
}
|
||||
return peerSet.ToSlice()[0]
|
||||
}
|
||||
@@ -509,21 +516,21 @@ func GetRemotePeers(endpoints EndpointList) []string {
|
||||
return peerSet.ToSlice()
|
||||
}
|
||||
|
||||
// In federated and distributed setup, update IP addresses of the hosts passed in command line
|
||||
// if MINIO_PUBLIC_IPS are not set manually
|
||||
func updateDomainIPs(endPoints set.StringSet) {
|
||||
_, dok := os.LookupEnv("MINIO_DOMAIN")
|
||||
_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
|
||||
_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if dok && eok && !iok {
|
||||
globalDomainIPs = set.NewStringSet()
|
||||
for e := range endPoints {
|
||||
host, _, _ := net.SplitHostPort(e)
|
||||
ipList, _ := getHostIP4(host)
|
||||
remoteIPList := ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.")
|
||||
}, "")
|
||||
globalDomainIPs.Add(remoteIPList.ToSlice()[0])
|
||||
ipList := set.NewStringSet()
|
||||
for e := range endPoints {
|
||||
host, _, err := net.SplitHostPort(e)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "missing port in address") {
|
||||
host = e
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
IPs, _ := getHostIP(host)
|
||||
ipList = ipList.Union(IPs)
|
||||
}
|
||||
globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
|
||||
}, "")
|
||||
}
|
||||
|
||||
@@ -304,27 +304,27 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
}, DistXLSetupType, nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
for i, testCase := range testCases {
|
||||
serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
|
||||
|
||||
if err == nil {
|
||||
if testCase.expectedErr != nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
t.Fatalf("Test (%d) error: expected = %v, got = <nil>", i+1, testCase.expectedErr)
|
||||
} else {
|
||||
if serverAddr != testCase.expectedServerAddr {
|
||||
t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
|
||||
t.Fatalf("Test (%d) serverAddr: expected = %v, got = %v", i+1, testCase.expectedServerAddr, serverAddr)
|
||||
}
|
||||
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
|
||||
t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
|
||||
t.Fatalf("Test (%d) endpoints: expected = %v, got = %v", i+1, testCase.expectedEndpoints, endpoints)
|
||||
}
|
||||
if setupType != testCase.expectedSetupType {
|
||||
t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
|
||||
t.Fatalf("Test (%d) setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
|
||||
}
|
||||
}
|
||||
} else if testCase.expectedErr == nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
t.Fatalf("Test (%d) error: expected = <nil>, got = %v", i+1, err)
|
||||
} else if err.Error() != testCase.expectedErr.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
t.Fatalf("Test (%d) error: expected = %v, got = %v", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
// Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvKMSMasterKey is the environment variable used to specify
|
||||
// a KMS master key used to protect SSE-S3 per-object keys.
|
||||
// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
|
||||
EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
|
||||
|
||||
// EnvAutoEncryption is the environment variable used to en/disable
|
||||
// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
|
||||
// requires a valid KMS configuration and turns any non-SSE-C
|
||||
// request into an SSE-S3 request.
|
||||
// If present EnvAutoEncryption must be either "on" or "off".
|
||||
EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvVaultEndpoint is the environment variable used to specify
|
||||
// the vault HTTPS endpoint.
|
||||
EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
|
||||
|
||||
// EnvVaultAuthType is the environment variable used to specify
|
||||
// the authentication type for vault.
|
||||
EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
|
||||
|
||||
// EnvVaultAppRoleID is the environment variable used to specify
|
||||
// the vault AppRole ID.
|
||||
EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
|
||||
|
||||
// EnvVaultAppSecretID is the environment variable used to specify
|
||||
// the vault AppRole secret corresponding to the AppRole ID.
|
||||
EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
|
||||
|
||||
// EnvVaultKeyVersion is the environment variable used to specify
|
||||
// the vault key version.
|
||||
EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
|
||||
|
||||
// EnvVaultKeyName is the environment variable used to specify
|
||||
// the vault named key-ring. In the S3 context it's referred as
|
||||
// customer master key ID (CMK-ID).
|
||||
EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
|
||||
|
||||
// EnvVaultCAPath is the environment variable used to specify the
|
||||
// path to a directory of PEM-encoded CA cert files. These CA cert
|
||||
// files are used to authenticate Minio to Vault over mTLS.
|
||||
EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
|
||||
|
||||
// EnvVaultNamespace is the environment variable used to specify
|
||||
// vault namespace. The vault namespace is used if the enterprise
|
||||
// version of Hashicorp Vault is used.
|
||||
EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
|
||||
)
|
||||
|
||||
// Environment provides functions for accessing environment
|
||||
// variables.
|
||||
var Environment = environment{}
|
||||
|
||||
type environment struct{}
|
||||
|
||||
// Get retrieves the value of the environment variable named
|
||||
// by the key. If the variable is present in the environment the
|
||||
// value (which may be empty) is returned. Otherwise it returns
|
||||
// the specified default value.
|
||||
func (environment) Get(key, defaultValue string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
// Lookup retrieves the value of the environment variable named
|
||||
// by the key. If the variable is present in the environment the
|
||||
// value (which may be empty) is returned and the boolean is true.
|
||||
// Otherwise the returned value will be empty and the boolean will
|
||||
// be false.
|
||||
func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) }
|
||||
|
||||
// LookupKMSConfig extracts the KMS configuration provided by environment
|
||||
// variables and merge them with the provided KMS configuration. The
|
||||
// merging follows the following rules:
|
||||
//
|
||||
// 1. A valid value provided as environment variable is higher prioritized
|
||||
// than the provided configuration and overwrites the value from the
|
||||
// configuration file.
|
||||
//
|
||||
// 2. A value specified as environment variable never changes the configuration
|
||||
// file. So it is never made a persistent setting.
|
||||
//
|
||||
// It sets the global KMS configuration according to the merged configuration
|
||||
// on success.
|
||||
func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) {
|
||||
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
|
||||
config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
|
||||
config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
|
||||
config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
|
||||
config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
|
||||
config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
|
||||
config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
|
||||
config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
|
||||
keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
|
||||
config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
|
||||
}
|
||||
if err = config.Vault.Verify(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Lookup KMS master keys - only available through ENV.
|
||||
if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
|
||||
if !config.Vault.IsEmpty() { // Vault and KMS master key provided
|
||||
return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
|
||||
}
|
||||
globalKMSKeyID, globalKMS, err = parseKMSMasterKey(masterKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !config.Vault.IsEmpty() {
|
||||
globalKMS, err = crypto.NewVault(config.Vault)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalKMSKeyID = config.Vault.Key.Name
|
||||
}
|
||||
|
||||
autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalAutoEncryption = bool(autoEncryption)
|
||||
if globalAutoEncryption && globalKMS == nil { // auto-encryption enabled but no KMS
|
||||
return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseKMSMasterKey parses the value of the environment variable
|
||||
// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
|
||||
func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) {
|
||||
values := strings.SplitN(envArg, ":", 2)
|
||||
if len(values) != 2 {
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
|
||||
}
|
||||
var (
|
||||
keyID = values[0]
|
||||
hexKey = values[1]
|
||||
)
|
||||
if len(hexKey) != 64 { // 2 hex bytes = 1 byte
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
|
||||
}
|
||||
var masterKey [32]byte
|
||||
if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
|
||||
}
|
||||
return keyID, crypto.NewKMS(masterKey), nil
|
||||
}
|
||||
@@ -381,6 +381,7 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
|
||||
|
||||
var ignoredHiddenDirectories = []string{
|
||||
minioMetaBucket,
|
||||
".snapshot",
|
||||
"lost+found",
|
||||
"$RECYCLE.BIN",
|
||||
"System Volume Information",
|
||||
|
||||
@@ -682,6 +682,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
fsRemoveAll(ctx, uploadIDDir)
|
||||
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
||||
fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
|
||||
fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
@@ -769,6 +771,9 @@ func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupIn
|
||||
}
|
||||
if now.Sub(fi.ModTime()) > expiry {
|
||||
fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
|
||||
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
||||
fsRemoveDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+17
-16
@@ -187,14 +187,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
case <-doneCh:
|
||||
return errWalkAbort
|
||||
default:
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
}
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
|
||||
@@ -226,14 +221,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
}
|
||||
}
|
||||
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
}
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
usage = usage + uint64(fi.Size())
|
||||
@@ -1222,9 +1212,8 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
|
||||
fs.listPool.Set(params, walkResultCh, endWalkCh)
|
||||
}
|
||||
|
||||
result := ListObjectsInfo{IsTruncated: !eof}
|
||||
result := ListObjectsInfo{}
|
||||
for _, objInfo := range objInfos {
|
||||
result.NextMarker = objInfo.Name
|
||||
if objInfo.IsDir && delimiter == slashSeparator {
|
||||
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
||||
continue
|
||||
@@ -1232,6 +1221,13 @@ func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, de
|
||||
result.Objects = append(result.Objects, objInfo)
|
||||
}
|
||||
|
||||
if !eof {
|
||||
result.IsTruncated = true
|
||||
if len(objInfos) > 0 {
|
||||
result.NextMarker = objInfos[len(objInfos)-1].Name
|
||||
}
|
||||
}
|
||||
|
||||
// Success.
|
||||
return result, nil
|
||||
}
|
||||
@@ -1316,6 +1312,11 @@ func (fs *FSObjects) IsNotificationSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (fs *FSObjects) IsListenBucketSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
func (fs *FSObjects) IsEncryptionSupported() bool {
|
||||
return true
|
||||
|
||||
+52
-48
@@ -18,8 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -112,39 +112,17 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if jsonFlag {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if quietFlag {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Fetch address option
|
||||
gatewayAddr := ctx.GlobalString("address")
|
||||
if gatewayAddr == ":"+globalMinioPort {
|
||||
gatewayAddr = ctx.String("address")
|
||||
}
|
||||
|
||||
// Handle common command args.
|
||||
handleCommonCmdArgs(ctx)
|
||||
|
||||
// Get port to listen on from gateway address
|
||||
_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
|
||||
if pErr != nil {
|
||||
logger.FatalIf(pErr, "Unable to start gateway")
|
||||
}
|
||||
globalMinioHost, globalMinioPort = mustSplitHostPort(globalCLIContext.Addr)
|
||||
|
||||
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
|
||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error situation we check for port availability.
|
||||
logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
|
||||
logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the gateway")
|
||||
|
||||
// Create certs path.
|
||||
logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
|
||||
@@ -176,11 +154,12 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
if globalEtcdClient != nil {
|
||||
// Enable STS router if etcd is enabled.
|
||||
registerSTSRouter(router)
|
||||
|
||||
// Enable admin router if etcd is enabled.
|
||||
registerAdminRouter(router)
|
||||
}
|
||||
|
||||
// Enable IAM admin APIs if etcd is enabled, if not just enable basic
|
||||
// operations such as profiling, server info etc.
|
||||
registerAdminRouter(router, globalEtcdClient != nil)
|
||||
|
||||
// Add healthcheck router
|
||||
registerHealthCheckRouter(router)
|
||||
|
||||
@@ -195,12 +174,21 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
// Add API router.
|
||||
registerAPIRouter(router)
|
||||
|
||||
// Dummy endpoint representing gateway instance.
|
||||
globalEndpoints = []Endpoint{{
|
||||
URL: &url.URL{Path: "/minio/gateway"},
|
||||
IsLocal: true,
|
||||
}}
|
||||
|
||||
// Initialize Admin Peers.
|
||||
initGlobalAdminPeers(globalEndpoints)
|
||||
|
||||
var getCert certs.GetCertificateFunc
|
||||
if globalTLSCerts != nil {
|
||||
getCert = globalTLSCerts.GetCertificate
|
||||
}
|
||||
|
||||
globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
|
||||
globalHTTPServer = xhttp.NewServer([]string{globalCLIContext.Addr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
|
||||
globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
|
||||
globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
|
||||
go func() {
|
||||
@@ -209,6 +197,22 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
// !!! Do not move this block !!!
|
||||
// For all gateways, the config needs to be loaded from env
|
||||
// prior to initializing the gateway layer
|
||||
{
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// Override any values from ENVs.
|
||||
srvCfg.loadFromEnvs()
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
}
|
||||
|
||||
newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
|
||||
if err != nil {
|
||||
// Stop watching for any certificate changes.
|
||||
@@ -218,23 +222,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
logger.FatalIf(err, "Unable to initialize gateway backend")
|
||||
}
|
||||
|
||||
// Create a new config system.
|
||||
globalConfigSys = NewConfigSys()
|
||||
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// Override any values from ENVs.
|
||||
srvCfg.loadFromEnvs()
|
||||
|
||||
// Load values to cached global values.
|
||||
srvCfg.loadToCachedConfigs()
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
if globalEtcdClient != nil && gatewayName == "nas" {
|
||||
// Create a new config system.
|
||||
globalConfigSys = NewConfigSys()
|
||||
|
||||
// Load globalServerConfig from etcd
|
||||
_ = globalConfigSys.Init(newObject)
|
||||
}
|
||||
// Load logger subsystem
|
||||
loadLoggers()
|
||||
|
||||
@@ -256,7 +250,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
globalIAMSys = NewIAMSys()
|
||||
if globalEtcdClient != nil {
|
||||
// Initialize IAM sys.
|
||||
go globalIAMSys.Init(newObject)
|
||||
_ = globalIAMSys.Init(newObject)
|
||||
}
|
||||
|
||||
// Create new policy system.
|
||||
@@ -267,6 +261,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
// Create new notification system.
|
||||
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
|
||||
if globalEtcdClient != nil && newObject.IsNotificationSupported() {
|
||||
_ = globalNotificationSys.Init(newObject)
|
||||
}
|
||||
|
||||
if globalAutoEncryption && !newObject.IsEncryptionSupported() {
|
||||
logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but gateway does not support encryption")
|
||||
}
|
||||
|
||||
// Once endpoints are finalized, initialize the new object api.
|
||||
globalObjLayerMutex.Lock()
|
||||
@@ -274,7 +275,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
if !quietFlag {
|
||||
if !globalCLIContext.Quiet {
|
||||
mode := globalMinioModeGatewayPrefix + gatewayName
|
||||
// Check update mode.
|
||||
checkUpdate(mode)
|
||||
@@ -285,8 +286,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
}
|
||||
|
||||
// Print gateway startup message.
|
||||
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
|
||||
printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
|
||||
}
|
||||
|
||||
// Set uptime time after object layer has initialized.
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
handleSignals()
|
||||
}
|
||||
|
||||
@@ -55,12 +55,14 @@ func printGatewayCommonMsg(apiEndpoints []string) {
|
||||
cred := globalServerConfig.GetCredential()
|
||||
|
||||
apiEndpointStr := strings.Join(apiEndpoints, " ")
|
||||
|
||||
// Colorize the message and print.
|
||||
logger.StartupMessage(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
if isTerminal() {
|
||||
logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
if isTerminal() && !globalCLIContext.Anonymous {
|
||||
logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
}
|
||||
printEventNotifiers()
|
||||
|
||||
if globalIsBrowserEnabled {
|
||||
logger.StartupMessage(colorBlue("\nBrowser Access:"))
|
||||
|
||||
@@ -132,6 +132,11 @@ func (a GatewayUnsupported) IsNotificationSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (a GatewayUnsupported) IsListenBucketSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
func (a GatewayUnsupported) IsEncryptionSupported() bool {
|
||||
return false
|
||||
|
||||
@@ -206,11 +206,10 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
|
||||
gcs := &gcsGateway{
|
||||
client: client,
|
||||
projectID: g.projectID,
|
||||
ctx: ctx,
|
||||
}
|
||||
|
||||
// Start background process to cleanup old files in minio.sys.tmp
|
||||
go gcs.CleanupGCSMinioSysTmp()
|
||||
go gcs.CleanupGCSMinioSysTmp(ctx)
|
||||
return gcs, nil
|
||||
}
|
||||
|
||||
@@ -349,7 +348,6 @@ type gcsGateway struct {
|
||||
minio.GatewayUnsupported
|
||||
client *storage.Client
|
||||
projectID string
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// Returns projectID from the GOOGLE_APPLICATION_CREDENTIALS file.
|
||||
@@ -366,8 +364,8 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
|
||||
}
|
||||
|
||||
// Cleanup old files in minio.sys.tmp of the given bucket.
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) {
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err != nil {
|
||||
@@ -380,7 +378,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
}
|
||||
if time.Since(attrs.Updated) > gcsMultipartExpiry {
|
||||
// Delete files older than 2 weeks.
|
||||
err := l.client.Bucket(bucket).Object(attrs.Name).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Object(attrs.Name).Delete(ctx)
|
||||
if err != nil {
|
||||
reqInfo := &logger.ReqInfo{BucketName: bucket, ObjectName: attrs.Name}
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
@@ -392,9 +390,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
}
|
||||
|
||||
// Cleanup old files in minio.sys.tmp of all buckets.
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmp() {
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmp(ctx context.Context) {
|
||||
for {
|
||||
it := l.client.Buckets(l.ctx, l.projectID)
|
||||
it := l.client.Buckets(ctx, l.projectID)
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err != nil {
|
||||
@@ -404,7 +402,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() {
|
||||
}
|
||||
break
|
||||
}
|
||||
l.CleanupGCSMinioSysTmpBucket(attrs.Name)
|
||||
l.CleanupGCSMinioSysTmpBucket(ctx, attrs.Name)
|
||||
}
|
||||
// Run the cleanup loop every 1 day.
|
||||
time.Sleep(gcsCleanupInterval)
|
||||
@@ -431,7 +429,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
|
||||
location = "us"
|
||||
}
|
||||
|
||||
err := bkt.Create(l.ctx, l.projectID, &storage.BucketAttrs{
|
||||
err := bkt.Create(ctx, l.projectID, &storage.BucketAttrs{
|
||||
Location: location,
|
||||
})
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -440,7 +438,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
|
||||
|
||||
// GetBucketInfo - Get bucket metadata..
|
||||
func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) {
|
||||
attrs, err := l.client.Bucket(bucket).Attrs(l.ctx)
|
||||
attrs, err := l.client.Bucket(bucket).Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.BucketInfo{}, gcsToObjectError(err, bucket)
|
||||
@@ -454,7 +452,7 @@ func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.Bu
|
||||
|
||||
// ListBuckets lists all buckets under your project-id on GCS.
|
||||
func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
|
||||
it := l.client.Buckets(l.ctx, l.projectID)
|
||||
it := l.client.Buckets(ctx, l.projectID)
|
||||
|
||||
// Iterate and capture all the buckets.
|
||||
for {
|
||||
@@ -479,7 +477,7 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
|
||||
|
||||
// DeleteBucket delete a bucket on GCS.
|
||||
func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
itObject := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
itObject := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: "/",
|
||||
Versions: false,
|
||||
})
|
||||
@@ -509,7 +507,7 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
}
|
||||
if gcsMinioPathFound {
|
||||
// Remove minio.sys.tmp before deleting the bucket.
|
||||
itObject = l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
|
||||
itObject = l.client.Bucket(bucket).Objects(ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
|
||||
for {
|
||||
objAttrs, err := itObject.Next()
|
||||
if err == iterator.Done {
|
||||
@@ -519,14 +517,14 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err)
|
||||
}
|
||||
err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(l.ctx)
|
||||
err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
err := l.client.Bucket(bucket).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Delete(ctx)
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -561,7 +559,7 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
|
||||
return minio.ListObjectsInfo{}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: delimiter,
|
||||
Prefix: prefix,
|
||||
Versions: false,
|
||||
@@ -668,7 +666,7 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
|
||||
return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: delimiter,
|
||||
Prefix: prefix,
|
||||
Versions: false,
|
||||
@@ -769,7 +767,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string,
|
||||
func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -781,7 +779,7 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s
|
||||
// Calling ReadCompressed with true accomplishes that.
|
||||
object := l.client.Bucket(bucket).Object(key).ReadCompressed(true)
|
||||
|
||||
r, err := object.NewRangeReader(l.ctx, startOffset, length)
|
||||
r, err := object.NewRangeReader(ctx, startOffset, length)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, key)
|
||||
@@ -864,12 +862,12 @@ func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAt
|
||||
func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
attrs, err := l.client.Bucket(bucket).Object(object).Attrs(l.ctx)
|
||||
attrs, err := l.client.Bucket(bucket).Object(object).Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, object)
|
||||
@@ -884,14 +882,14 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
|
||||
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
object := l.client.Bucket(bucket).Object(key)
|
||||
|
||||
w := object.NewWriter(l.ctx)
|
||||
w := object.NewWriter(ctx)
|
||||
// Disable "chunked" uploading in GCS client if the size of the data to be uploaded is below
|
||||
// the current chunk-size of the writer. This avoids an unnecessary memory allocation.
|
||||
if data.Size() < int64(w.ChunkSize) {
|
||||
@@ -909,7 +907,7 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
|
||||
// Close the object writer upon success.
|
||||
w.Close()
|
||||
|
||||
attrs, err := object.Attrs(l.ctx)
|
||||
attrs, err := object.Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -928,7 +926,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
|
||||
copier := dst.CopierFrom(src)
|
||||
applyMetadataToGCSAttrs(srcInfo.UserDefined, &copier.ObjectAttrs)
|
||||
|
||||
attrs, err := copier.Run(l.ctx)
|
||||
attrs, err := copier.Run(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, destBucket, destObject)
|
||||
@@ -939,7 +937,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
|
||||
|
||||
// DeleteObject - Deletes a blob in bucket
|
||||
func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error {
|
||||
err := l.client.Bucket(bucket).Object(object).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Object(object).Delete(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, object)
|
||||
@@ -956,7 +954,7 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
|
||||
// generate name for part zero
|
||||
meta := gcsMultipartMetaName(uploadID)
|
||||
|
||||
w := l.client.Bucket(bucket).Object(meta).NewWriter(l.ctx)
|
||||
w := l.client.Bucket(bucket).Object(meta).NewWriter(ctx)
|
||||
defer w.Close()
|
||||
|
||||
applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
|
||||
@@ -1056,7 +1054,7 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr
|
||||
// Checks if minio.sys.tmp/multipart/v1/<upload-id>/gcs.json exists, returns
|
||||
// an object layer compatible error upon any error.
|
||||
func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error {
|
||||
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx)
|
||||
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(ctx)
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, key, uploadID)
|
||||
}
|
||||
@@ -1073,7 +1071,7 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
|
||||
etag = minio.GenETag()
|
||||
}
|
||||
object := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, partNumber, etag))
|
||||
w := object.NewWriter(l.ctx)
|
||||
w := object.NewWriter(ctx)
|
||||
// Disable "chunked" uploading in GCS client. If enabled, it can cause a corner case
|
||||
// where it tries to upload 0 bytes in the last chunk and get error from server.
|
||||
w.ChunkSize = 0
|
||||
@@ -1186,7 +1184,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
|
||||
prefix := fmt.Sprintf("%s/%s/", gcsMinioMultipartPathV1, uploadID)
|
||||
|
||||
// iterate through all parts and delete them
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: prefix, Versions: false})
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: prefix, Versions: false})
|
||||
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
@@ -1200,7 +1198,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
|
||||
|
||||
object := l.client.Bucket(bucket).Object(attrs.Name)
|
||||
// Ignore the error as parallel AbortMultipartUpload might have deleted it.
|
||||
object.Delete(l.ctx)
|
||||
object.Delete(ctx)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1222,13 +1220,13 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
meta := gcsMultipartMetaName(uploadID)
|
||||
object := l.client.Bucket(bucket).Object(meta)
|
||||
|
||||
partZeroAttrs, err := object.Attrs(l.ctx)
|
||||
partZeroAttrs, err := object.Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key, uploadID)
|
||||
}
|
||||
|
||||
r, err := object.NewReader(l.ctx)
|
||||
r, err := object.NewReader(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -1259,7 +1257,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
for i, uploadedPart := range uploadedParts {
|
||||
parts = append(parts, l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID,
|
||||
uploadedPart.PartNumber, uploadedPart.ETag)))
|
||||
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(l.ctx)
|
||||
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(ctx)
|
||||
if pErr != nil {
|
||||
logger.LogIf(ctx, pErr)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(pErr, bucket, key, uploadID)
|
||||
@@ -1305,7 +1303,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
composer.ContentType = partZeroAttrs.ContentType
|
||||
composer.Metadata = partZeroAttrs.Metadata
|
||||
|
||||
if _, err = composer.Run(l.ctx); err != nil {
|
||||
if _, err = composer.Run(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
}
|
||||
@@ -1322,7 +1320,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
composer.ContentDisposition = partZeroAttrs.ContentDisposition
|
||||
composer.ContentLanguage = partZeroAttrs.ContentLanguage
|
||||
composer.Metadata = partZeroAttrs.Metadata
|
||||
attrs, err := composer.Run(l.ctx)
|
||||
attrs, err := composer.Run(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -1362,7 +1360,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
|
||||
acl := l.client.Bucket(bucket).ACL()
|
||||
if policies[0].Policy == miniogopolicy.BucketPolicyNone {
|
||||
if err := acl.Delete(l.ctx, storage.AllUsers); err != nil {
|
||||
if err := acl.Delete(ctx, storage.AllUsers); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1380,7 +1378,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
return minio.NotImplemented{}
|
||||
}
|
||||
|
||||
if err := acl.Set(l.ctx, storage.AllUsers, role); err != nil {
|
||||
if err := acl.Set(ctx, storage.AllUsers, role); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1390,7 +1388,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
|
||||
// GetBucketPolicy - Get policy on bucket
|
||||
func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
|
||||
rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
|
||||
rules, err := l.client.Bucket(bucket).ACL().List(ctx)
|
||||
if err != nil {
|
||||
return nil, gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1449,7 +1447,7 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
|
||||
// DeleteBucketPolicy - Delete all policies on bucket
|
||||
func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
||||
// This only removes the storage.AllUsers policies
|
||||
if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
|
||||
if err := l.client.Bucket(bucket).ACL().Delete(ctx, storage.AllUsers); err != nil {
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ func (g *NAS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &nasObjects{newObject.(*minio.FSObjects)}, nil
|
||||
return &nasObjects{newObject}, nil
|
||||
}
|
||||
|
||||
// Production - nas gateway is production ready.
|
||||
@@ -116,17 +116,12 @@ func (g *NAS) Production() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this gateway.
|
||||
func (n *nasObjects) IsListenBucketSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// nasObjects implements gateway for Minio and S3 compatible object storage servers.
|
||||
type nasObjects struct {
|
||||
*minio.FSObjects
|
||||
}
|
||||
|
||||
// IsNotificationSupported returns whether notifications are applicable for this layer.
|
||||
func (l *nasObjects) IsNotificationSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsCompressionSupported returns whether compression is applicable for this layer.
|
||||
func (l *nasObjects) IsCompressionSupported() bool {
|
||||
return false
|
||||
minio.ObjectLayer
|
||||
}
|
||||
|
||||
@@ -510,12 +510,12 @@ func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, e error) {
|
||||
err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
|
||||
etag, err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
|
||||
if err != nil {
|
||||
return oi, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
|
||||
return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
|
||||
return minio.ObjectInfo{Bucket: bucket, Name: object, ETag: etag}, nil
|
||||
}
|
||||
|
||||
// SetBucketPolicy sets policy on bucket
|
||||
|
||||
+65
-21
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -91,7 +90,7 @@ func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
|
||||
// of the user-defined metadata to 2 KB.
|
||||
func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if isHTTPHeaderSizeTooLarge(r.Header) {
|
||||
writeErrorResponse(w, ErrMetadataTooLarge, r.URL)
|
||||
writeErrorResponse(w, ErrMetadataTooLarge, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
h.Handler.ServeHTTP(w, r)
|
||||
@@ -134,7 +133,7 @@ func filterReservedMetadata(h http.Handler) http.Handler {
|
||||
// would be treated as metadata.
|
||||
func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if containsReservedMetadata(r.Header) {
|
||||
writeErrorResponse(w, ErrUnsupportedMetadata, r.URL)
|
||||
writeErrorResponse(w, ErrUnsupportedMetadata, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
h.Handler.ServeHTTP(w, r)
|
||||
@@ -228,7 +227,8 @@ func guessIsRPCReq(req *http.Request) bool {
|
||||
if req == nil {
|
||||
return false
|
||||
}
|
||||
return req.Method == http.MethodPost
|
||||
return req.Method == http.MethodPost &&
|
||||
strings.HasPrefix(req.URL.Path, minioReservedBucketPath+"/")
|
||||
}
|
||||
|
||||
func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -300,7 +300,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
|
||||
// buckets
|
||||
bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
|
||||
if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
|
||||
writeErrorResponse(w, ErrAllAccessDisabled, r.URL)
|
||||
writeErrorResponse(w, ErrAllAccessDisabled, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -363,14 +363,14 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// All our internal APIs are sensitive towards Date
|
||||
// header, for all requests where Date header is not
|
||||
// present we will reject such clients.
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
|
||||
// or in the future, reject request otherwise.
|
||||
curTime := UTCNow()
|
||||
if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
|
||||
writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL)
|
||||
writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -485,20 +485,20 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// If bucketName is present and not objectName check for bucket level resource queries.
|
||||
if bucketName != "" && objectName == "" {
|
||||
if ignoreNotImplementedBucketResources(r) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
// If bucketName and objectName are present check for its resource queries.
|
||||
if bucketName != "" && objectName != "" {
|
||||
if ignoreNotImplementedObjectResources(r) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
// A put method on path "/" doesn't make sense, ignore it.
|
||||
if r.Method == http.MethodPut && r.URL.Path == "/" {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -601,14 +601,14 @@ func hasBadPathComponent(path string) bool {
|
||||
func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Check for bad components in URL path.
|
||||
if hasBadPathComponent(r.URL.Path) {
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Check for bad components in URL query values.
|
||||
for _, vv := range r.URL.Query() {
|
||||
for _, v := range vv {
|
||||
if hasBadPathComponent(v) {
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -625,10 +625,53 @@ type bucketForwardingHandler struct {
|
||||
}
|
||||
|
||||
func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if globalDNSConfig == nil || globalDomainName == "" || guessIsBrowserReq(r) || guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || guessIsRPCReq(r) {
|
||||
if globalDNSConfig == nil || globalDomainName == "" ||
|
||||
guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
|
||||
guessIsRPCReq(r) || isAdminReq(r) {
|
||||
f.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// For browser requests, when federation is setup we need to
|
||||
// specifically handle download and upload for browser requests.
|
||||
if guessIsBrowserReq(r) && globalDNSConfig != nil && globalDomainName != "" {
|
||||
var bucket, _ string
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
if getRequestAuthType(r) == authTypeJWT {
|
||||
bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/upload"))
|
||||
}
|
||||
case http.MethodGet:
|
||||
if t := r.URL.Query().Get("token"); t != "" {
|
||||
bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/download"))
|
||||
} else if getRequestAuthType(r) != authTypeJWT && !strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
|
||||
bucket, _ = urlPath2BucketObjectName(r.URL.Path)
|
||||
}
|
||||
}
|
||||
if bucket != "" {
|
||||
sr, err := globalDNSConfig.Get(bucket)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
|
||||
} else {
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
r.URL.Scheme = "http"
|
||||
if globalIsSSL {
|
||||
r.URL.Scheme = "https"
|
||||
}
|
||||
r.URL.Host = getHostFromSrv(sr)
|
||||
f.fwd.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
f.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
bucket, object := urlPath2BucketObjectName(r.URL.Path)
|
||||
// ListBucket requests should be handled at current endpoint as
|
||||
// all buckets data can be fetched from here.
|
||||
@@ -653,19 +696,18 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
|
||||
sr, err := globalDNSConfig.Get(bucket)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL)
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
|
||||
} else {
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
host, port := getRandomHostPort(sr)
|
||||
r.URL.Scheme = "http"
|
||||
if globalIsSSL {
|
||||
r.URL.Scheme = "https"
|
||||
}
|
||||
r.URL.Host = fmt.Sprintf("%s:%d", host, port)
|
||||
r.URL.Host = getHostFromSrv(sr)
|
||||
f.fwd.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
@@ -716,7 +758,7 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// potential pileup on the server.
|
||||
if err := l.Wait(ctx); err != nil {
|
||||
// Send an S3 compatible error, SlowDown.
|
||||
writeErrorResponse(w, ErrSlowDown, r.URL)
|
||||
writeErrorResponse(w, ErrSlowDown, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -740,7 +782,9 @@ func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Set custom headers such as x-amz-request-id and x-minio-deployment-id
|
||||
// for each request.
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
|
||||
w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
|
||||
if globalDeploymentID != "" {
|
||||
w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
|
||||
}
|
||||
s.handler.ServeHTTP(logger.NewResponseWriter(w), r)
|
||||
}
|
||||
|
||||
@@ -768,7 +812,7 @@ type criticalErrorHandler struct{ handler http.Handler }
|
||||
func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err == logger.ErrCritical { // handle
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
} else if err != nil {
|
||||
panic(err) // forward other panic calls
|
||||
}
|
||||
@@ -787,7 +831,7 @@ func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead {
|
||||
writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
|
||||
} else {
|
||||
writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL)
|
||||
writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -78,9 +78,16 @@ func TestGuessIsRPC(t *testing.T) {
|
||||
if guessIsRPCReq(nil) {
|
||||
t.Fatal("Unexpected return for nil request")
|
||||
}
|
||||
|
||||
u, err := url.Parse("http://localhost:9000/minio/lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := &http.Request{
|
||||
Proto: "HTTP/1.0",
|
||||
Method: http.MethodPost,
|
||||
URL: u,
|
||||
}
|
||||
if !guessIsRPCReq(r) {
|
||||
t.Fatal("Test shouldn't fail for a possible net/rpc request.")
|
||||
|
||||
+14
-3
@@ -42,6 +42,8 @@ import (
|
||||
const (
|
||||
globalMinioCertExpireWarnDays = time.Hour * 24 * 30 // 30 days.
|
||||
|
||||
globalMinioDefaultPort = "9000"
|
||||
|
||||
globalMinioDefaultRegion = ""
|
||||
// This is a sha256 output of ``arn:aws:iam::minio:user/admin``,
|
||||
// this is kept in present form to be compatible with S3 owner ID
|
||||
@@ -90,6 +92,12 @@ const (
|
||||
maxLocationConstraintSize = 3 * humanize.MiByte
|
||||
)
|
||||
|
||||
var globalCLIContext = struct {
|
||||
JSON, Quiet bool
|
||||
Anonymous bool
|
||||
Addr string
|
||||
}{}
|
||||
|
||||
var (
|
||||
// Indicates the total number of erasure coded sets configured.
|
||||
globalXLSetCount int
|
||||
@@ -127,7 +135,7 @@ var (
|
||||
// Minio local server address (in `host:port` format)
|
||||
globalMinioAddr = ""
|
||||
// Minio default port, can be changed through command line.
|
||||
globalMinioPort = "9000"
|
||||
globalMinioPort = globalMinioDefaultPort
|
||||
// Holds the host that was passed using --address
|
||||
globalMinioHost = ""
|
||||
|
||||
@@ -230,8 +238,11 @@ var (
|
||||
globalKMSKeyID string
|
||||
// Allocated KMS
|
||||
globalKMS crypto.KMS
|
||||
// KMS config
|
||||
globalKMSConfig crypto.KMSConfig
|
||||
|
||||
// Auto-Encryption, if enabled, turns any non-SSE-C request
|
||||
// into an SSE-S3 request. If enabled a valid, non-empty KMS
|
||||
// configuration must be present.
|
||||
globalAutoEncryption bool
|
||||
|
||||
// Is compression include extensions/content-types set.
|
||||
globalIsEnvCompression bool
|
||||
|
||||
+29
-8
@@ -28,6 +28,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
httptracer "github.com/minio/minio/pkg/handlers"
|
||||
)
|
||||
@@ -133,6 +134,11 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set content-type to default value if it is not set.
|
||||
if _, ok := metadata["content-type"]; !ok {
|
||||
metadata["content-type"] = "application/octet-stream"
|
||||
}
|
||||
|
||||
// Success.
|
||||
return metadata, nil
|
||||
}
|
||||
@@ -176,13 +182,20 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
|
||||
return redirectValues.Encode()
|
||||
}
|
||||
|
||||
// Returns access key in the request Authorization header.
|
||||
func getReqAccessKey(r *http.Request, region string) (accessKey string) {
|
||||
cred, _, _ := getReqAccessKeyV4(r, region)
|
||||
// Returns access credentials in the request Authorization header.
|
||||
func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
|
||||
cred, _, _ = getReqAccessKeyV4(r, region)
|
||||
if cred.AccessKey == "" {
|
||||
cred, _, _ = getReqAccessKeyV2(r)
|
||||
}
|
||||
return cred.AccessKey
|
||||
if cred.AccessKey == "" {
|
||||
claims, owner, _ := webRequestAuthenticate(r)
|
||||
if owner {
|
||||
return globalServerConfig.GetCredential()
|
||||
}
|
||||
cred, _ = globalIAMSys.GetUser(claims.Subject)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
// Extract request params to be sent with event notifiation.
|
||||
@@ -192,10 +205,12 @@ func extractReqParams(r *http.Request) map[string]string {
|
||||
}
|
||||
|
||||
region := globalServerConfig.GetRegion()
|
||||
cred := getReqAccessCred(r, region)
|
||||
|
||||
// Success.
|
||||
return map[string]string{
|
||||
"region": region,
|
||||
"accessKey": getReqAccessKey(r, region),
|
||||
"accessKey": cred.AccessKey,
|
||||
"sourceIPAddress": handlers.GetSourceIP(r),
|
||||
// Add more fields here.
|
||||
}
|
||||
@@ -349,8 +364,14 @@ func getResource(path string, host string, domain string) (string, error) {
|
||||
return slashSeparator + pathJoin(bucket, path), nil
|
||||
}
|
||||
|
||||
// If none of the http routes match respond with MethodNotAllowed
|
||||
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
// If none of the http routes match respond with MethodNotAllowed, in JSON
|
||||
func notFoundHandlerJSON(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// If none of the http routes match respond with MethodNotAllowed
|
||||
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
+2
-2
@@ -99,10 +99,10 @@ func (c *BufConn) Write(b []byte) (n int, err error) {
|
||||
}
|
||||
|
||||
// newBufConn - creates a new connection object wrapping net.Conn.
|
||||
func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration) *BufConn {
|
||||
func newBufConn(c net.Conn, readTimeout, writeTimeout time.Duration, maxHeaderBytes int) *BufConn {
|
||||
return &BufConn{
|
||||
QuirkConn: QuirkConn{Conn: c},
|
||||
bufReader: bufio.NewReader(c),
|
||||
bufReader: bufio.NewReaderSize(c, maxHeaderBytes),
|
||||
readTimeout: readTimeout,
|
||||
writeTimeout: writeTimeout,
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ func TestBuffConnReadTimeout(t *testing.T) {
|
||||
t.Errorf("failed to accept new connection. %v", terr)
|
||||
return
|
||||
}
|
||||
bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second)
|
||||
bufconn := newBufConn(tcpConn, 1*time.Second, 1*time.Second, 4096)
|
||||
defer bufconn.Close()
|
||||
|
||||
// Read a line
|
||||
|
||||
@@ -178,6 +178,9 @@ type httpListener struct {
|
||||
// isRoutineNetErr returns true if error is due to a network timeout,
|
||||
// connect reset or io.EOF and false otherwise
|
||||
func isRoutineNetErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if nErr, ok := err.(*net.OpError); ok {
|
||||
// Check if the error is a tcp connection reset
|
||||
if syscallErr, ok := nErr.Err.(*os.SyscallError); ok {
|
||||
@@ -188,7 +191,8 @@ func isRoutineNetErr(err error) bool {
|
||||
// Check if the error is a timeout
|
||||
return nErr.Timeout()
|
||||
}
|
||||
return err == io.EOF
|
||||
// check for io.EOF and also some times io.EOF is wrapped is another error type.
|
||||
return err == io.EOF || err.Error() == "EOF"
|
||||
}
|
||||
|
||||
// start - starts separate goroutine for each TCP listener. A valid insecure/TLS HTTP new connection is passed to httpListener.acceptCh.
|
||||
@@ -218,7 +222,7 @@ func (listener *httpListener) start() {
|
||||
tcpConn.SetKeepAlive(true)
|
||||
tcpConn.SetKeepAlivePeriod(listener.tcpKeepAliveTimeout)
|
||||
|
||||
bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout)
|
||||
bufconn := newBufConn(tcpConn, listener.readTimeout, listener.writeTimeout, listener.maxHeaderBytes)
|
||||
if listener.tlsConfig != nil {
|
||||
ok, err := getPlainText(bufconn)
|
||||
if err != nil {
|
||||
@@ -257,7 +261,7 @@ func (listener *httpListener) start() {
|
||||
return
|
||||
}
|
||||
|
||||
bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout)
|
||||
bufconn = newBufConn(tlsConn, listener.readTimeout, listener.writeTimeout, listener.maxHeaderBytes)
|
||||
}
|
||||
|
||||
method, resource, host, err := getMethodResourceHost(bufconn, listener.maxHeaderBytes)
|
||||
@@ -395,8 +399,8 @@ func newHTTPListener(serverAddrs []string,
|
||||
|
||||
for _, serverAddr := range serverAddrs {
|
||||
var l net.Listener
|
||||
if l, err = listen("tcp4", serverAddr); err != nil {
|
||||
if l, err = fallbackListen("tcp4", serverAddr); err != nil {
|
||||
if l, err = listen("tcp", serverAddr); err != nil {
|
||||
if l, err = fallbackListen("tcp", serverAddr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
@@ -801,6 +802,10 @@ func TestIgnoreErr(t *testing.T) {
|
||||
err: &net.OpError{Err: &myTimeoutErr{timeout: true}},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
err: errors.New("EOF"),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
err: &net.OpError{Err: &myTimeoutErr{timeout: false}},
|
||||
want: false,
|
||||
@@ -809,6 +814,10 @@ func TestIgnoreErr(t *testing.T) {
|
||||
err: io.ErrUnexpectedEOF,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
err: nil,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ const (
|
||||
DefaultShutdownTimeout = 5 * time.Second
|
||||
|
||||
// DefaultTCPKeepAliveTimeout - default TCP keep alive timeout for accepted connection.
|
||||
DefaultTCPKeepAliveTimeout = 10 * time.Second
|
||||
DefaultTCPKeepAliveTimeout = 30 * time.Second
|
||||
|
||||
// DefaultReadTimeout - default timout to read data from accepted connection.
|
||||
DefaultReadTimeout = 5 * time.Minute
|
||||
|
||||
+2
-2
@@ -60,9 +60,9 @@ type IAMSys struct {
|
||||
iamCannedPolicyMap map[string]iampolicy.Policy
|
||||
}
|
||||
|
||||
// Load - load iam.json
|
||||
// Load - loads iam subsystem
|
||||
func (sys *IAMSys) Load(objAPI ObjectLayer) error {
|
||||
return sys.Init(objAPI)
|
||||
return sys.refresh(objAPI)
|
||||
}
|
||||
|
||||
// Init - initializes config system from iam.json
|
||||
|
||||
@@ -18,9 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"io/ioutil"
|
||||
@@ -76,15 +74,6 @@ func (lc localAdminClient) ServerInfo() (sid ServerInfoData, e error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetConfig - returns config.json of the local server.
|
||||
func (lc localAdminClient) GetConfig() ([]byte, error) {
|
||||
if globalServerConfig == nil {
|
||||
return nil, fmt.Errorf("config not present")
|
||||
}
|
||||
|
||||
return json.Marshal(globalServerConfig)
|
||||
}
|
||||
|
||||
// StartProfiling - starts profiling on the local server.
|
||||
func (lc localAdminClient) StartProfiling(profiler string) error {
|
||||
if globalProfiler != nil {
|
||||
|
||||
@@ -24,14 +24,6 @@ func TestLocalAdminClientSignalService(t *testing.T) {
|
||||
testAdminCmdRunnerSignalService(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
func TestLocalAdminClientReInitFormat(t *testing.T) {
|
||||
testAdminCmdRunnerReInitFormat(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
func TestLocalAdminClientServerInfo(t *testing.T) {
|
||||
testAdminCmdRunnerServerInfo(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
func TestLocalAdminClientGetConfig(t *testing.T) {
|
||||
testAdminCmdRunnerGetConfig(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
+2
-2
@@ -55,7 +55,7 @@ func AddAuditTarget(t Target) {
|
||||
}
|
||||
|
||||
// AuditLog - logs audit logs to all audit targets.
|
||||
func AuditLog(w http.ResponseWriter, r *http.Request, api string) {
|
||||
func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}) {
|
||||
var statusCode int
|
||||
lrw, ok := w.(*ResponseWriter)
|
||||
if ok {
|
||||
@@ -63,6 +63,6 @@ func AuditLog(w http.ResponseWriter, r *http.Request, api string) {
|
||||
}
|
||||
// Send audit logs only to http targets.
|
||||
for _, t := range AuditTargets {
|
||||
t.Send(audit.ToEntry(w, r, api, statusCode))
|
||||
_ = t.Send(audit.ToEntry(w, r, api, statusCode, reqClaims))
|
||||
}
|
||||
}
|
||||
|
||||
+36
-1
@@ -18,17 +18,28 @@ package logger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"go/build"
|
||||
"hash"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/highwayhash"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger/message/log"
|
||||
)
|
||||
|
||||
var (
|
||||
// HighwayHash key for logging in anonymous mode
|
||||
magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
|
||||
// HighwayHash hasher for logging in anonymous mode
|
||||
loggerHighwayHasher hash.Hash
|
||||
)
|
||||
|
||||
// Disable disables all logging, false by default. (used for "go test")
|
||||
var Disable = false
|
||||
|
||||
@@ -94,7 +105,7 @@ func (level Level) String() string {
|
||||
// quietFlag: Hide startup messages if enabled
|
||||
// jsonFlag: Display in JSON format, if enabled
|
||||
var (
|
||||
quietFlag, jsonFlag bool
|
||||
quietFlag, jsonFlag, anonFlag bool
|
||||
// Custom function to format error
|
||||
errorFmtFunc func(string, error, bool) string
|
||||
)
|
||||
@@ -110,6 +121,12 @@ func EnableJSON() {
|
||||
quietFlag = true
|
||||
}
|
||||
|
||||
// EnableAnonymous - turns anonymous flag
|
||||
// to avoid printing sensitive information.
|
||||
func EnableAnonymous() {
|
||||
anonFlag = true
|
||||
}
|
||||
|
||||
// IsJSON - returns true if jsonFlag is true
|
||||
func IsJSON() bool {
|
||||
return jsonFlag
|
||||
@@ -187,6 +204,8 @@ func Init(goPath string, goRoot string) {
|
||||
// paths like "{GOROOT}/src/github.com/minio/minio"
|
||||
// and "{GOPATH}/src/github.com/minio/minio"
|
||||
trimStrings = append(trimStrings, filepath.Join("github.com", "minio", "minio")+string(filepath.Separator))
|
||||
|
||||
loggerHighwayHasher, _ = highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
|
||||
}
|
||||
|
||||
func trimTrace(f string) string {
|
||||
@@ -239,6 +258,14 @@ func getTrace(traceLevel int) []string {
|
||||
return trace
|
||||
}
|
||||
|
||||
// Return the highway hash of the passed string
|
||||
func hashString(input string) string {
|
||||
defer loggerHighwayHasher.Reset()
|
||||
loggerHighwayHasher.Write([]byte(input))
|
||||
checksum := loggerHighwayHasher.Sum(nil)
|
||||
return hex.EncodeToString(checksum)
|
||||
}
|
||||
|
||||
// LogAlwaysIf prints a detailed error message during
|
||||
// the execution of the server.
|
||||
func LogAlwaysIf(ctx context.Context, err error) {
|
||||
@@ -312,6 +339,14 @@ func logIf(ctx context.Context, err error) {
|
||||
},
|
||||
}
|
||||
|
||||
if anonFlag {
|
||||
entry.API.Args.Bucket = hashString(entry.API.Args.Bucket)
|
||||
entry.API.Args.Object = hashString(entry.API.Args.Object)
|
||||
entry.RemoteHost = hashString(entry.RemoteHost)
|
||||
entry.Message = reflect.TypeOf(err).String()
|
||||
entry.Trace.Variables = make(map[string]string)
|
||||
}
|
||||
|
||||
// Iterate over all logger targets to send the log entry
|
||||
for _, t := range Targets {
|
||||
t.Send(entry)
|
||||
|
||||
@@ -40,16 +40,17 @@ type Entry struct {
|
||||
Status string `json:"status,omitempty"`
|
||||
StatusCode int `json:"statusCode,omitempty"`
|
||||
} `json:"api"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
ReqQuery map[string]string `json:"requestQuery,omitempty"`
|
||||
ReqHeader map[string]string `json:"requestHeader,omitempty"`
|
||||
RespHeader map[string]string `json:"responseHeader,omitempty"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
ReqClaims map[string]interface{} `json:"requestClaims,omitempty"`
|
||||
ReqQuery map[string]string `json:"requestQuery,omitempty"`
|
||||
ReqHeader map[string]string `json:"requestHeader,omitempty"`
|
||||
RespHeader map[string]string `json:"responseHeader,omitempty"`
|
||||
}
|
||||
|
||||
// ToEntry - constructs an audit entry object.
|
||||
func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int) Entry {
|
||||
func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int, reqClaims map[string]interface{}) Entry {
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
@@ -77,6 +78,7 @@ func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int)
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
ReqQuery: reqQuery,
|
||||
ReqHeader: reqHeader,
|
||||
ReqClaims: reqClaims,
|
||||
RespHeader: respHeader,
|
||||
}
|
||||
|
||||
|
||||
@@ -44,6 +44,10 @@ var globalFlags = []cli.Flag{
|
||||
Name: "quiet",
|
||||
Usage: "Disable startup information.",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "anonymous",
|
||||
Usage: "Hide sensitive information from logging.",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "json",
|
||||
Usage: "Output server logs and startup information in json format.",
|
||||
|
||||
+57
-20
@@ -37,14 +37,21 @@ import (
|
||||
// IPv4 addresses of local host.
|
||||
var localIP4 = mustGetLocalIP4()
|
||||
|
||||
// IPv6 address of local host.
|
||||
var localIP6 = mustGetLocalIP6()
|
||||
|
||||
// mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal.
|
||||
func mustSplitHostPort(hostPort string) (host, port string) {
|
||||
host, port, err := net.SplitHostPort(hostPort)
|
||||
// Strip off IPv6 zone information.
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
logger.FatalIf(err, "Unable to split host port %s", hostPort)
|
||||
return host, port
|
||||
}
|
||||
|
||||
// mustGetLocalIP4 returns IPv4 addresses of local host. It panics on error.
|
||||
// mustGetLocalIP4 returns IPv4 addresses of localhost. It panics on error.
|
||||
func mustGetLocalIP4() (ipList set.StringSet) {
|
||||
ipList = set.NewStringSet()
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
@@ -67,8 +74,31 @@ func mustGetLocalIP4() (ipList set.StringSet) {
|
||||
return ipList
|
||||
}
|
||||
|
||||
// getHostIP4 returns IPv4 address of given host.
|
||||
func getHostIP4(host string) (ipList set.StringSet, err error) {
|
||||
// mustGetLocalIP6 returns IPv6 addresses of localhost. It panics on error.
|
||||
func mustGetLocalIP6() (ipList set.StringSet) {
|
||||
ipList = set.NewStringSet()
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
logger.FatalIf(err, "Unable to get IP addresses of this host")
|
||||
|
||||
for _, addr := range addrs {
|
||||
var ip net.IP
|
||||
switch v := addr.(type) {
|
||||
case *net.IPNet:
|
||||
ip = v.IP
|
||||
case *net.IPAddr:
|
||||
ip = v.IP
|
||||
}
|
||||
|
||||
if ip.To4() == nil {
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
}
|
||||
|
||||
return ipList
|
||||
}
|
||||
|
||||
// getHostIP returns IP address of given host.
|
||||
func getHostIP(host string) (ipList set.StringSet, err error) {
|
||||
var ips []net.IP
|
||||
|
||||
if ips, err = net.LookupIP(host); err != nil {
|
||||
@@ -110,9 +140,7 @@ func getHostIP4(host string) (ipList set.StringSet, err error) {
|
||||
|
||||
ipList = set.NewStringSet()
|
||||
for _, ip := range ips {
|
||||
if ip.To4() != nil {
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
|
||||
return ipList, err
|
||||
@@ -169,29 +197,32 @@ func sortIPs(ipList []string) []string {
|
||||
return append(nonIPs, ips...)
|
||||
}
|
||||
|
||||
func getAPIEndpoints(serverAddr string) (apiEndpoints []string) {
|
||||
host, port := mustSplitHostPort(serverAddr)
|
||||
|
||||
func getAPIEndpoints() (apiEndpoints []string) {
|
||||
var ipList []string
|
||||
if host == "" {
|
||||
if globalMinioHost == "" {
|
||||
ipList = sortIPs(localIP4.ToSlice())
|
||||
ipList = append(ipList, localIP6.ToSlice()...)
|
||||
} else {
|
||||
ipList = []string{host}
|
||||
ipList = []string{globalMinioHost}
|
||||
}
|
||||
|
||||
for _, ip := range ipList {
|
||||
apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), ip, port))
|
||||
apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(ip, globalMinioPort)))
|
||||
}
|
||||
|
||||
return apiEndpoints
|
||||
}
|
||||
|
||||
// isHostIPv4 - helper for validating if the provided arg is an ip address.
|
||||
func isHostIPv4(ipAddress string) bool {
|
||||
// isHostIP - helper for validating if the provided arg is an ip address.
|
||||
func isHostIP(ipAddress string) bool {
|
||||
host, _, err := net.SplitHostPort(ipAddress)
|
||||
if err != nil {
|
||||
host = ipAddress
|
||||
}
|
||||
// Strip off IPv6 zone information.
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
return net.ParseIP(host) != nil
|
||||
}
|
||||
|
||||
@@ -291,19 +322,20 @@ func extractHostPort(hostAddr string) (string, string, error) {
|
||||
// correspond to one of the local IP of the
|
||||
// current machine
|
||||
func isLocalHost(host string) (bool, error) {
|
||||
hostIPs, err := getHostIP4(host)
|
||||
hostIPs, err := getHostIP(host)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// If intersection of two IP sets is not empty, then the host is local host.
|
||||
isLocal := !localIP4.Intersection(hostIPs).IsEmpty()
|
||||
return isLocal, nil
|
||||
// If intersection of two IP sets is not empty, then the host is localhost.
|
||||
isLocalv4 := !localIP4.Intersection(hostIPs).IsEmpty()
|
||||
isLocalv6 := !localIP6.Intersection(hostIPs).IsEmpty()
|
||||
return isLocalv4 || isLocalv6, nil
|
||||
}
|
||||
|
||||
// sameLocalAddrs - returns true if two addresses, even with different
|
||||
// formats, point to the same machine, e.g:
|
||||
// ':9000' and 'http://localhost:9000/' will return true
|
||||
// ':9000' and 'http://localhost:9000/' will return true
|
||||
func sameLocalAddrs(addr1, addr2 string) (bool, error) {
|
||||
|
||||
// Extract host & port from given parameters
|
||||
@@ -355,6 +387,11 @@ func CheckLocalServerAddr(serverAddr string) error {
|
||||
return uiErrInvalidAddressFlag(err)
|
||||
}
|
||||
|
||||
// Strip off IPv6 zone information.
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
|
||||
// Check whether port is a valid port number.
|
||||
p, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
@@ -366,7 +403,7 @@ func CheckLocalServerAddr(serverAddr string) error {
|
||||
// 0.0.0.0 is a wildcard address and refers to local network
|
||||
// addresses. I.e, 0.0.0.0:9000 like ":9000" refers to port
|
||||
// 9000 on localhost.
|
||||
if host != "" && host != net.IPv4zero.String() {
|
||||
if host != "" && host != net.IPv4zero.String() && host != net.IPv6zero.String() {
|
||||
isLocalHost, err := isLocalHost(host)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+14
-8
@@ -138,7 +138,7 @@ func TestGetHostIP(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
ipList, err := getHostIP4(testCase.host)
|
||||
ipList, err := getHostIP(testCase.host)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
@@ -157,17 +157,22 @@ func TestGetHostIP(t *testing.T) {
|
||||
|
||||
// Tests finalize api endpoints.
|
||||
func TestGetAPIEndpoints(t *testing.T) {
|
||||
host, port := globalMinioHost, globalMinioAddr
|
||||
defer func() {
|
||||
globalMinioHost, globalMinioAddr = host, port
|
||||
}()
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
host, port string
|
||||
expectedResult string
|
||||
}{
|
||||
{":80", "http://127.0.0.1:80"},
|
||||
{"127.0.0.1:80", "http://127.0.0.1:80"},
|
||||
{"localhost:80", "http://localhost:80"},
|
||||
{"", "80", "http://127.0.0.1:80"},
|
||||
{"127.0.0.1", "80", "http://127.0.0.1:80"},
|
||||
{"localhost", "80", "http://localhost:80"},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
apiEndpoints := getAPIEndpoints(testCase.serverAddr)
|
||||
globalMinioHost, globalMinioPort = testCase.host, testCase.port
|
||||
apiEndpoints := getAPIEndpoints()
|
||||
apiEndpointSet := set.CreateStringSet(apiEndpoints...)
|
||||
if !apiEndpointSet.Contains(testCase.expectedResult) {
|
||||
t.Fatalf("test %d: expected: Found, got: Not Found", i+1)
|
||||
@@ -330,7 +335,7 @@ func TestSameLocalAddrs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestIsHostIPv4(t *testing.T) {
|
||||
func TestIsHostIP(t *testing.T) {
|
||||
testCases := []struct {
|
||||
args string
|
||||
expectedResult bool
|
||||
@@ -341,10 +346,11 @@ func TestIsHostIPv4(t *testing.T) {
|
||||
{"http://192.168.1.0", false},
|
||||
{"http://192.168.1.0:9000", false},
|
||||
{"192.168.1.0", true},
|
||||
{"[2001:3984:3989::20%eth0]:9000", true},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
ret := isHostIPv4(testCase.args)
|
||||
ret := isHostIP(testCase.args)
|
||||
if testCase.expectedResult != ret {
|
||||
t.Fatalf("expected: %v , got: %v", testCase.expectedResult, ret)
|
||||
}
|
||||
|
||||
+65
-3
@@ -22,6 +22,7 @@ import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -89,6 +90,60 @@ func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string)
|
||||
}()
|
||||
}
|
||||
|
||||
// ReloadFormat - calls ReloadFormat RPC call on all peers.
|
||||
func (sys *NotificationSys) ReloadFormat(dryRun bool) map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
var wg sync.WaitGroup
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to load format in three attempts, before giving up.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.ReloadFormat(dryRun)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
errors[addr] = err
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
}(addr, client)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
// LoadUsers - calls LoadUsers RPC call on all peers.
|
||||
func (sys *NotificationSys) LoadUsers() map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
var wg sync.WaitGroup
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to load users in three attempts.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.LoadUsers()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
errors[addr] = err
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
}(addr, client)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
// LoadCredentials - calls LoadCredentials RPC call on all peers.
|
||||
func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
@@ -97,7 +152,7 @@ func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to set credentials in three attempts.
|
||||
// Try to load credentials in three attempts.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.LoadCredentials()
|
||||
if err == nil {
|
||||
@@ -488,10 +543,10 @@ func (args eventArgs) ToEvent() event.Event {
|
||||
host := globalMinioHost
|
||||
if host == "" {
|
||||
// FIXME: Send FQDN or hostname of this machine than sending IP address.
|
||||
host = localIP4.ToSlice()[0]
|
||||
host = sortIPs(localIP4.ToSlice())[0]
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), host, globalMinioPort)
|
||||
return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
|
||||
}
|
||||
|
||||
eventTime := UTCNow()
|
||||
@@ -501,6 +556,10 @@ func (args eventArgs) ToEvent() event.Event {
|
||||
"x-amz-request-id": args.RespElements["requestId"],
|
||||
"x-minio-origin-endpoint": getOriginEndpoint(), // Minio specific custom elements.
|
||||
}
|
||||
// Add deployment as part of
|
||||
if globalDeploymentID != "" {
|
||||
respElements["x-minio-deployment-id"] = globalDeploymentID
|
||||
}
|
||||
if args.RespElements["content-length"] != "" {
|
||||
respElements["content-length"] = args.RespElements["content-length"]
|
||||
}
|
||||
@@ -537,6 +596,9 @@ func (args eventArgs) ToEvent() event.Event {
|
||||
if args.EventName != event.ObjectRemovedDelete {
|
||||
newEvent.S3.Object.ETag = args.Object.ETag
|
||||
newEvent.S3.Object.Size = args.Object.Size
|
||||
if args.Object.IsCompressed() {
|
||||
newEvent.S3.Object.Size = args.Object.GetActualSize()
|
||||
}
|
||||
newEvent.S3.Object.ContentType = args.Object.ContentType
|
||||
newEvent.S3.Object.UserMetadata = args.Object.UserDefined
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ func toObjectErr(err error, params ...string) error {
|
||||
Object: params[1],
|
||||
}
|
||||
}
|
||||
case errIsNotRegular, errFileAccessDenied:
|
||||
case errIsNotRegular:
|
||||
if len(params) >= 2 {
|
||||
err = ObjectExistsAsDirectory{
|
||||
Bucket: params[0],
|
||||
|
||||
@@ -94,6 +94,7 @@ type ObjectLayer interface {
|
||||
|
||||
// Supported operations check
|
||||
IsNotificationSupported() bool
|
||||
IsListenBucketSupported() bool
|
||||
IsEncryptionSupported() bool
|
||||
|
||||
// Compression support check.
|
||||
|
||||
@@ -574,6 +574,14 @@ func testListObjects(obj ObjectLayer, instanceType string, t TestErrHandler) {
|
||||
t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated)
|
||||
}
|
||||
|
||||
if testCase.result.IsTruncated && result.NextMarker == "" {
|
||||
t.Errorf("Test %d: %s: Expected NextContinuationToken to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType)
|
||||
}
|
||||
|
||||
if !testCase.result.IsTruncated && result.NextMarker != "" {
|
||||
t.Errorf("Test %d: %s: Expected NextContinuationToken to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker)
|
||||
}
|
||||
|
||||
}
|
||||
// Take ListObject treeWalk go-routine to completion, if available in the treewalk pool.
|
||||
if result.IsTruncated {
|
||||
|
||||
+15
-9
@@ -23,6 +23,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"path"
|
||||
"runtime"
|
||||
@@ -299,11 +300,11 @@ func getHostsSlice(records []dns.SrvRecord) []string {
|
||||
return hosts
|
||||
}
|
||||
|
||||
// returns a random host (and corresponding port) from a slice of DNS records
|
||||
func getRandomHostPort(records []dns.SrvRecord) (string, int) {
|
||||
// returns a host (and corresponding port) from a slice of DNS records
|
||||
func getHostFromSrv(records []dns.SrvRecord) string {
|
||||
rand.Seed(time.Now().Unix())
|
||||
srvRecord := records[rand.Intn(len(records))]
|
||||
return srvRecord.Host, srvRecord.Port
|
||||
return net.JoinHostPort(srvRecord.Host, fmt.Sprintf("%d", srvRecord.Port))
|
||||
}
|
||||
|
||||
// IsCompressed returns true if the object is marked as compressed.
|
||||
@@ -502,6 +503,9 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Decrypt the ETag before top layer consumes this value.
|
||||
oi.ETag = getDecryptedETag(h, oi, copySource)
|
||||
|
||||
// Apply the skipLen and limit on the
|
||||
// decrypted stream
|
||||
decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength)
|
||||
@@ -635,12 +639,13 @@ func (p *PutObjReader) MD5CurrentHexString() string {
|
||||
func NewPutObjReader(rawReader *hash.Reader, encReader *hash.Reader, encKey []byte) *PutObjReader {
|
||||
p := PutObjReader{Reader: rawReader, rawReader: rawReader}
|
||||
|
||||
var objKey crypto.ObjectKey
|
||||
copy(objKey[:], encKey)
|
||||
p.sealMD5Fn = sealETagFn(objKey)
|
||||
if encReader != nil {
|
||||
if len(encKey) != 0 && encReader != nil {
|
||||
var objKey crypto.ObjectKey
|
||||
copy(objKey[:], encKey)
|
||||
p.sealMD5Fn = sealETagFn(objKey)
|
||||
p.Reader = encReader
|
||||
}
|
||||
|
||||
return &p
|
||||
}
|
||||
|
||||
@@ -651,9 +656,10 @@ func sealETag(encKey crypto.ObjectKey, md5CurrSum []byte) []byte {
|
||||
}
|
||||
return encKey.SealETag(md5CurrSum)
|
||||
}
|
||||
|
||||
func sealETagFn(key crypto.ObjectKey) SealMD5CurrFn {
|
||||
fn1 := func(md5sumcurr []byte) []byte {
|
||||
fn := func(md5sumcurr []byte) []byte {
|
||||
return sealETag(key, md5sumcurr)
|
||||
}
|
||||
return fn1
|
||||
return fn
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if !ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
// If the object is not modified since the specified time.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
// If the object is modified since the specified time.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if objInfo.ETag != "" && !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
|
||||
// If the object ETag does not match with the specified ETag.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if objInfo.ETag != "" && isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
|
||||
// If the object ETag matches with the specified ETag.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -174,7 +174,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
|
||||
if ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
// If the object is modified since the specified time.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -187,7 +187,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
|
||||
if !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
|
||||
// If the object ETag does not match with the specified ETag.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
+362
-249
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,7 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -1811,9 +1812,13 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
|
||||
// this is required even to assert the copied object,
|
||||
bytesData := []struct {
|
||||
byteData []byte
|
||||
md5sum string
|
||||
}{
|
||||
{generateBytesData(6 * humanize.KiByte)},
|
||||
{byteData: generateBytesData(6 * humanize.KiByte)},
|
||||
}
|
||||
h := md5.New()
|
||||
h.Write(bytesData[0].byteData)
|
||||
bytesData[0].md5sum = hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
buffers := []*bytes.Buffer{
|
||||
new(bytes.Buffer),
|
||||
@@ -1826,23 +1831,29 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
|
||||
objectName string
|
||||
contentLength int64
|
||||
textData []byte
|
||||
md5sum string
|
||||
metaData map[string]string
|
||||
}{
|
||||
// case - 1.
|
||||
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
|
||||
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)},
|
||||
|
||||
// case - 2.
|
||||
// used for anonymous HTTP request test.
|
||||
{bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
|
||||
{bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)},
|
||||
}
|
||||
|
||||
// iterate through the above set of inputs and upload the object.
|
||||
for i, input := range putObjectInputs {
|
||||
// uploading the object.
|
||||
_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
|
||||
// if object upload fails stop the test.
|
||||
if err != nil {
|
||||
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)
|
||||
}
|
||||
if objInfo.ETag != input.md5sum {
|
||||
t.Fatalf("Put Object case %d: Checksum mismatched: <ERROR> got %s, expected %s", i+1, input.md5sum, objInfo.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
// test cases with inputs and expected result for Copy Object.
|
||||
@@ -2105,6 +2116,16 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
|
||||
t.Fatalf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
|
||||
}
|
||||
if rec.Code == http.StatusOK {
|
||||
var cpObjResp CopyObjectResponse
|
||||
if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil {
|
||||
t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i+1, instanceType, err)
|
||||
}
|
||||
|
||||
retag := canonicalizeETag(cpObjResp.ETag)
|
||||
if retag != bytesData[0].md5sum {
|
||||
t.Fatalf("Test %d: %s: Failed to CopyObject: <ERROR> got %s, expected %s", i+1, instanceType, retag, bytesData[0].md5sum)
|
||||
}
|
||||
|
||||
// See if the new object is formed.
|
||||
// testing whether the copy was successful.
|
||||
err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
|
||||
|
||||
+19
-1
@@ -115,12 +115,30 @@ func (rpcClient *PeerRPCClient) SendEvent(bucketName string, targetID, remoteTar
|
||||
return err
|
||||
}
|
||||
|
||||
// ReloadFormat - calls reload format RPC.
|
||||
func (rpcClient *PeerRPCClient) ReloadFormat(dryRun bool) error {
|
||||
args := ReloadFormatArgs{
|
||||
DryRun: dryRun,
|
||||
}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(peerServiceName+".ReloadFormat", &args, &reply)
|
||||
}
|
||||
|
||||
// LoadUsers - calls load users RPC.
|
||||
func (rpcClient *PeerRPCClient) LoadUsers() error {
|
||||
args := AuthArgs{}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(peerServiceName+".LoadUsers", &args, &reply)
|
||||
}
|
||||
|
||||
// LoadCredentials - calls load credentials RPC.
|
||||
func (rpcClient *PeerRPCClient) LoadCredentials() error {
|
||||
args := AuthArgs{}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(peerServiceName+".SetCredentials", &args, &reply)
|
||||
return rpcClient.Call(peerServiceName+".LoadCredentials", &args, &reply)
|
||||
}
|
||||
|
||||
// NewPeerRPCClient - returns new peer RPC client.
|
||||
|
||||
@@ -192,6 +192,30 @@ func (receiver *peerRPCReceiver) SendEvent(args *SendEventArgs, reply *bool) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReloadFormatArgs - send event RPC arguments.
|
||||
type ReloadFormatArgs struct {
|
||||
AuthArgs
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// ReloadFormat - handles reload format RPC call, reloads latest `format.json`
|
||||
func (receiver *peerRPCReceiver) ReloadFormat(args *ReloadFormatArgs, reply *VoidReply) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
return objAPI.ReloadFormat(context.Background(), args.DryRun)
|
||||
}
|
||||
|
||||
// LoadUsers - handles load users RPC call.
|
||||
func (receiver *peerRPCReceiver) LoadUsers(args *AuthArgs, reply *VoidReply) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
return globalIAMSys.Load(objAPI)
|
||||
}
|
||||
|
||||
// LoadCredentials - handles load credentials RPC call.
|
||||
func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidReply) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
|
||||
+7
-3
@@ -20,6 +20,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -183,7 +184,12 @@ func NewPolicySys() *PolicySys {
|
||||
}
|
||||
|
||||
func getConditionValues(request *http.Request, locationConstraint string) map[string][]string {
|
||||
args := make(map[string][]string)
|
||||
args := map[string][]string{
|
||||
"SourceIp": {handlers.GetSourceIP(request)},
|
||||
"SecureTransport": {fmt.Sprintf("%t", request.TLS != nil)},
|
||||
"UserAgent": {request.UserAgent()},
|
||||
"Referer": {request.Referer()},
|
||||
}
|
||||
|
||||
for key, values := range request.Header {
|
||||
if existingValues, found := args[key]; found {
|
||||
@@ -201,8 +207,6 @@ func getConditionValues(request *http.Request, locationConstraint string) map[st
|
||||
}
|
||||
}
|
||||
|
||||
args["SourceIp"] = []string{handlers.GetSourceIP(request)}
|
||||
|
||||
if locationConstraint != "" {
|
||||
args["LocationConstraint"] = []string{locationConstraint}
|
||||
}
|
||||
|
||||
+24
-4
@@ -71,6 +71,7 @@ type posix struct {
|
||||
connected bool
|
||||
|
||||
diskMount bool // indicates if the path is an actual mount.
|
||||
driveSync bool // indicates if the backend is synchronous.
|
||||
|
||||
// Disk usage metrics
|
||||
stopUsageCh chan struct{}
|
||||
@@ -83,8 +84,10 @@ func checkPathLength(pathName string) error {
|
||||
return errFileNameTooLong
|
||||
}
|
||||
|
||||
// Convert any '\' to '/'.
|
||||
pathName = filepath.ToSlash(pathName)
|
||||
if runtime.GOOS == "windows" {
|
||||
// Convert any '\' to '/'.
|
||||
pathName = filepath.ToSlash(pathName)
|
||||
}
|
||||
|
||||
// Check each path segment length is > 255
|
||||
for len(pathName) > 0 && pathName != "." && pathName != "/" {
|
||||
@@ -189,6 +192,15 @@ func newPosix(path string) (*posix, error) {
|
||||
diskMount: mountinfo.IsLikelyMountPoint(path),
|
||||
}
|
||||
|
||||
var pf BoolFlag
|
||||
if driveSync := os.Getenv("MINIO_DRIVE_SYNC"); driveSync != "" {
|
||||
pf, err = ParseBoolFlag(driveSync)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.driveSync = bool(pf)
|
||||
}
|
||||
|
||||
if !p.diskMount {
|
||||
go p.diskUsage(globalServiceDoneCh)
|
||||
}
|
||||
@@ -374,6 +386,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
default:
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
atomic.AddUint64(&s.totalUsed, uint64(fi.Size()))
|
||||
@@ -413,6 +426,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
default:
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
usage = usage + uint64(fi.Size())
|
||||
@@ -1009,8 +1023,14 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
|
||||
return errFaultyDisk
|
||||
}
|
||||
|
||||
// Create file if not found
|
||||
w, err := s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
|
||||
var w *os.File
|
||||
// Create file if not found, additionally also enables synchronous
|
||||
// operation if asked by the user.
|
||||
if s.driveSync {
|
||||
w, err = s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_APPEND|os.O_WRONLY)
|
||||
} else {
|
||||
w, err = s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -128,6 +128,15 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
|
||||
}
|
||||
defer closeStorageDisks(storageDisks)
|
||||
|
||||
// Attempt to load all `format.json` from all disks.
|
||||
formatConfigs, sErrs := loadFormatXLAll(storageDisks)
|
||||
// Check if we have
|
||||
for i, sErr := range sErrs {
|
||||
if _, ok := formatCriticalErrors[sErr]; ok {
|
||||
return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Connect to all storage disks, a connection failure will be
|
||||
// only logged after some retries.
|
||||
for _, disk := range storageDisks {
|
||||
@@ -139,15 +148,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to load all `format.json` from all disks.
|
||||
formatConfigs, sErrs := loadFormatXLAll(storageDisks)
|
||||
// Check if we have
|
||||
for i, sErr := range sErrs {
|
||||
if _, ok := formatCriticalErrors[sErr]; ok {
|
||||
return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-emptively check if one of the formatted disks
|
||||
// is invalid. This function returns success for the
|
||||
// most part unless one of the formats is not consistent
|
||||
|
||||
+2
-2
@@ -107,8 +107,8 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
|
||||
// Add Admin RPC router
|
||||
registerAdminRPCRouter(router)
|
||||
|
||||
// Add Admin router.
|
||||
registerAdminRouter(router)
|
||||
// Add Admin router, all APIs are enabled in server mode.
|
||||
registerAdminRouter(router, true)
|
||||
|
||||
// Add healthcheck router
|
||||
registerHealthCheckRouter(router)
|
||||
|
||||
+11
-23
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -40,7 +41,7 @@ func init() {
|
||||
var serverFlags = []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "address",
|
||||
Value: ":" + globalMinioPort,
|
||||
Value: ":" + globalMinioDefaultPort,
|
||||
Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
|
||||
},
|
||||
}
|
||||
@@ -147,9 +148,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
|
||||
// Handle common command args.
|
||||
handleCommonCmdArgs(ctx)
|
||||
|
||||
// Server address.
|
||||
serverAddr := ctx.String("address")
|
||||
logger.FatalIf(CheckLocalServerAddr(serverAddr), "Unable to validate passed arguments")
|
||||
logger.FatalIf(CheckLocalServerAddr(globalCLIContext.Addr), "Unable to validate passed arguments")
|
||||
|
||||
var setupType SetupType
|
||||
var err error
|
||||
@@ -162,9 +161,9 @@ func serverHandleCmdArgs(ctx *cli.Context) {
|
||||
|
||||
endpoints := strings.Fields(os.Getenv("MINIO_ENDPOINTS"))
|
||||
if len(endpoints) > 0 {
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, endpoints...)
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
|
||||
} else {
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, ctx.Args()...)
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, ctx.Args()...)
|
||||
}
|
||||
logger.FatalIf(err, "Invalid command line arguments")
|
||||
|
||||
@@ -205,19 +204,6 @@ func serverMain(ctx *cli.Context) {
|
||||
// error during initialization will be shown as a fatal message
|
||||
logger.Disable = true
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if jsonFlag {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if quietFlag {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Handle all server command args.
|
||||
serverHandleCmdArgs(ctx)
|
||||
|
||||
@@ -246,7 +232,7 @@ func serverMain(ctx *cli.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
if !quietFlag {
|
||||
if !globalCLIContext.Quiet {
|
||||
// Check for new updates from dl.minio.io.
|
||||
mode := globalMinioModeFS
|
||||
if globalIsDistXL {
|
||||
@@ -381,7 +367,10 @@ func serverMain(ctx *cli.Context) {
|
||||
|
||||
// Initialize notification system.
|
||||
if err = globalNotificationSys.Init(newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize notification system")
|
||||
logger.LogIf(context.Background(), err)
|
||||
}
|
||||
if globalAutoEncryption && !newObject.IsEncryptionSupported() {
|
||||
logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but server does not support encryption")
|
||||
}
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
@@ -389,8 +378,7 @@ func serverMain(ctx *cli.Context) {
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
apiEndpoints := getAPIEndpoints(globalMinioAddr)
|
||||
printStartupMessage(apiEndpoints)
|
||||
printStartupMessage(getAPIEndpoints())
|
||||
|
||||
// Set uptime time after object layer has initialized.
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
+21
-14
@@ -20,12 +20,13 @@ import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"net"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
// Documentation links, these are part of message printing code.
|
||||
@@ -76,6 +77,19 @@ func printStartupMessage(apiEndPoints []string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if input is not IPv4, false if it is.
|
||||
func isNotIPv4(host string) bool {
|
||||
h, _, err := net.SplitHostPort(host)
|
||||
if err != nil {
|
||||
h = host
|
||||
}
|
||||
ip := net.ParseIP(h)
|
||||
ok := ip.To4() != nil // This is always true of IP is IPv4
|
||||
|
||||
// Returns true if input is not IPv4.
|
||||
return !ok
|
||||
}
|
||||
|
||||
// strip api endpoints list with standard ports such as
|
||||
// port "80" and "443" before displaying on the startup
|
||||
// banner. Returns a new list of API endpoints.
|
||||
@@ -83,23 +97,16 @@ func stripStandardPorts(apiEndpoints []string) (newAPIEndpoints []string) {
|
||||
newAPIEndpoints = make([]string, len(apiEndpoints))
|
||||
// Check all API endpoints for standard ports and strip them.
|
||||
for i, apiEndpoint := range apiEndpoints {
|
||||
url, err := url.Parse(apiEndpoint)
|
||||
u, err := xnet.ParseURL(apiEndpoint)
|
||||
if err != nil {
|
||||
newAPIEndpoints[i] = apiEndpoint
|
||||
continue
|
||||
}
|
||||
host, port := mustSplitHostPort(url.Host)
|
||||
// For standard HTTP(s) ports such as "80" and "443"
|
||||
// apiEndpoints should only be host without port.
|
||||
switch {
|
||||
case url.Scheme == "http" && port == "80":
|
||||
fallthrough
|
||||
case url.Scheme == "https" && port == "443":
|
||||
url.Host = host
|
||||
newAPIEndpoints[i] = url.String()
|
||||
default:
|
||||
newAPIEndpoints[i] = apiEndpoint
|
||||
if globalMinioHost == "" && isNotIPv4(u.Host) {
|
||||
// Skip all non-IPv4 endpoints when we bind to all interfaces.
|
||||
continue
|
||||
}
|
||||
newAPIEndpoints[i] = u.String()
|
||||
}
|
||||
return newAPIEndpoints
|
||||
}
|
||||
@@ -116,7 +123,7 @@ func printServerCommonMsg(apiEndpoints []string) {
|
||||
|
||||
// Colorize the message and print.
|
||||
logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
if isTerminal() {
|
||||
if isTerminal() && !globalCLIContext.Anonymous {
|
||||
logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
if region != "" {
|
||||
|
||||
@@ -28,6 +28,9 @@ const (
|
||||
|
||||
// DistXLSetupType - Distributed XL setup type enum.
|
||||
DistXLSetupType
|
||||
|
||||
// GatewaySetupType - gateway setup type enum.
|
||||
GatewaySetupType
|
||||
)
|
||||
|
||||
func (setupType SetupType) String() string {
|
||||
@@ -38,6 +41,8 @@ func (setupType SetupType) String() string {
|
||||
return globalMinioModeXL
|
||||
case DistXLSetupType:
|
||||
return globalMinioModeDistXL
|
||||
case GatewaySetupType:
|
||||
return globalMinioModeGatewayPrefix
|
||||
}
|
||||
|
||||
return ""
|
||||
|
||||
@@ -377,7 +377,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
|
||||
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
|
||||
Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
|
||||
|
||||
subrouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
}
|
||||
|
||||
router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
}
|
||||
|
||||
@@ -127,6 +127,8 @@ type ClientGrantsResult struct {
|
||||
func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AssumeRoleWithClientGrants")
|
||||
|
||||
defer logger.AuditLog(w, r, "AssumeRoleWithClientGrants", nil)
|
||||
|
||||
if globalIAMValidators == nil {
|
||||
writeSTSErrorResponse(w, ErrSTSNotInitialized)
|
||||
return
|
||||
|
||||
@@ -2319,7 +2319,7 @@ func getEndpointsLocalAddr(endpoints EndpointList) string {
|
||||
}
|
||||
}
|
||||
|
||||
return globalMinioHost + ":" + globalMinioPort
|
||||
return net.JoinHostPort(globalMinioHost, globalMinioPort)
|
||||
}
|
||||
|
||||
// fetches a random number between range min-max.
|
||||
|
||||
@@ -105,6 +105,12 @@ const (
|
||||
httpsScheme = "https"
|
||||
)
|
||||
|
||||
// nopCharsetConverter is a dummy charset convert which just copies input to output,
|
||||
// it is used to ignore custom encoding charset in S3 XML body.
|
||||
func nopCharsetConverter(label string, input io.Reader) (io.Reader, error) {
|
||||
return input, nil
|
||||
}
|
||||
|
||||
// xmlDecoder provide decoded value in xml.
|
||||
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
||||
var lbody io.Reader
|
||||
@@ -114,6 +120,8 @@ func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
||||
lbody = body
|
||||
}
|
||||
d := xml.NewDecoder(lbody)
|
||||
// Ignore any encoding set in the XML body
|
||||
d.CharsetReader = nopCharsetConverter
|
||||
return d.Decode(v)
|
||||
}
|
||||
|
||||
|
||||
+396
-163
@@ -99,11 +99,14 @@ func (web *webAPIHandlers) ServerInfo(r *http.Request, args *WebGenericArgs, rep
|
||||
reply.MinioGlobalInfo = getGlobalInfo()
|
||||
// If ENV creds are not set and incoming user is not owner
|
||||
// disable changing credentials.
|
||||
// TODO: fix this in future and allow changing user credentials.
|
||||
v, ok := reply.MinioGlobalInfo["isEnvCreds"].(bool)
|
||||
if ok && !v {
|
||||
reply.MinioGlobalInfo["isEnvCreds"] = !owner
|
||||
}
|
||||
// if etcd is set disallow changing credentials through UI
|
||||
if globalEtcdClient != nil {
|
||||
reply.MinioGlobalInfo["isEnvCreds"] = true
|
||||
}
|
||||
|
||||
reply.MinioMemory = mem
|
||||
reply.MinioPlatform = platform
|
||||
@@ -148,7 +151,7 @@ func (web *webAPIHandlers) MakeBucket(r *http.Request, args *MakeBucketArgs, rep
|
||||
if authErr != nil {
|
||||
return toJSONError(authErr)
|
||||
}
|
||||
// TODO: Allow MakeBucket in future.
|
||||
|
||||
if !owner {
|
||||
return toJSONError(errAccessDenied)
|
||||
}
|
||||
@@ -201,11 +204,33 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
|
||||
if authErr != nil {
|
||||
return toJSONError(authErr)
|
||||
}
|
||||
// TODO: Allow DeleteBucket in future.
|
||||
|
||||
if !owner {
|
||||
return toJSONError(errAccessDenied)
|
||||
}
|
||||
|
||||
reply.UIVersion = browser.UIVersion
|
||||
|
||||
if isRemoteCallRequired(context.Background(), args.BucketName, objectAPI) {
|
||||
sr, err := globalDNSConfig.Get(args.BucketName)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
return toJSONError(BucketNotFound{
|
||||
Bucket: args.BucketName,
|
||||
}, args.BucketName)
|
||||
}
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
core, err := getRemoteInstanceClient(r, getHostFromSrv(sr))
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
if err = core.RemoveBucket(args.BucketName); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
deleteBucket := objectAPI.DeleteBucket
|
||||
@@ -229,7 +254,6 @@ func (web *webAPIHandlers) DeleteBucket(r *http.Request, args *RemoveBucketArgs,
|
||||
}
|
||||
}
|
||||
|
||||
reply.UIVersion = browser.UIVersion
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -258,7 +282,8 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
|
||||
listBuckets = web.CacheAPI().ListBuckets
|
||||
}
|
||||
|
||||
if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
|
||||
claims, owner, authErr := webRequestAuthenticate(r)
|
||||
if authErr != nil {
|
||||
return toJSONError(authErr)
|
||||
}
|
||||
|
||||
@@ -270,10 +295,20 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
|
||||
}
|
||||
for _, dnsRecord := range dnsBuckets {
|
||||
bucketName := strings.Trim(dnsRecord.Key, "/")
|
||||
reply.Buckets = append(reply.Buckets, WebBucketInfo{
|
||||
Name: bucketName,
|
||||
CreationDate: dnsRecord.CreationDate,
|
||||
})
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.ListBucketAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
ObjectName: "",
|
||||
}) {
|
||||
reply.Buckets = append(reply.Buckets, WebBucketInfo{
|
||||
Name: bucketName,
|
||||
CreationDate: dnsRecord.CreationDate,
|
||||
})
|
||||
}
|
||||
}
|
||||
} else {
|
||||
buckets, err := listBuckets(context.Background())
|
||||
@@ -281,10 +316,19 @@ func (web *webAPIHandlers) ListBuckets(r *http.Request, args *WebGenericArgs, re
|
||||
return toJSONError(err)
|
||||
}
|
||||
for _, bucket := range buckets {
|
||||
reply.Buckets = append(reply.Buckets, WebBucketInfo{
|
||||
Name: bucket.Name,
|
||||
CreationDate: bucket.Created,
|
||||
})
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.ListBucketAction,
|
||||
BucketName: bucket.Name,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
ObjectName: "",
|
||||
}) {
|
||||
reply.Buckets = append(reply.Buckets, WebBucketInfo{
|
||||
Name: bucket.Name,
|
||||
CreationDate: bucket.Created,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,16 +377,56 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
|
||||
listObjects = web.CacheAPI().ListObjects
|
||||
}
|
||||
|
||||
if isRemoteCallRequired(context.Background(), args.BucketName, objectAPI) {
|
||||
sr, err := globalDNSConfig.Get(args.BucketName)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
return toJSONError(BucketNotFound{
|
||||
Bucket: args.BucketName,
|
||||
}, args.BucketName)
|
||||
}
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
core, err := getRemoteInstanceClient(r, getHostFromSrv(sr))
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
result, err := core.ListObjects(args.BucketName, args.Prefix, args.Marker, slashSeparator, 1000)
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
reply.NextMarker = result.NextMarker
|
||||
reply.IsTruncated = result.IsTruncated
|
||||
for _, obj := range result.Contents {
|
||||
reply.Objects = append(reply.Objects, WebObjectInfo{
|
||||
Key: obj.Key,
|
||||
LastModified: obj.LastModified,
|
||||
Size: obj.Size,
|
||||
ContentType: obj.ContentType,
|
||||
})
|
||||
}
|
||||
for _, p := range result.CommonPrefixes {
|
||||
reply.Objects = append(reply.Objects, WebObjectInfo{
|
||||
Key: p.Prefix,
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
claims, owner, authErr := webRequestAuthenticate(r)
|
||||
if authErr != nil {
|
||||
if authErr == errNoAuthToken {
|
||||
// Add this for checking ListObjects conditional.
|
||||
if args.Prefix != "" {
|
||||
r.Header.Set("prefix", args.Prefix)
|
||||
}
|
||||
|
||||
// Check if anonymous (non-owner) has access to download objects.
|
||||
readable := globalPolicySys.IsAllowed(policy.Args{
|
||||
Action: policy.GetObjectAction,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: args.BucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: false,
|
||||
ObjectName: args.Prefix + "/",
|
||||
})
|
||||
|
||||
// Check if anonymous (non-owner) has access to upload objects.
|
||||
@@ -370,18 +454,22 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
|
||||
|
||||
// For authenticated users apply IAM policy.
|
||||
if authErr == nil {
|
||||
// Add this for checking ListObjects conditional.
|
||||
if args.Prefix != "" {
|
||||
r.Header.Set("prefix", args.Prefix)
|
||||
}
|
||||
|
||||
readable := globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.Action(policy.GetObjectAction),
|
||||
Action: iampolicy.ListBucketAction,
|
||||
BucketName: args.BucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
ObjectName: args.Prefix + "/",
|
||||
})
|
||||
|
||||
writable := globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.Action(policy.PutObjectAction),
|
||||
Action: iampolicy.PutObjectAction,
|
||||
BucketName: args.BucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
@@ -465,6 +553,40 @@ func (web *webAPIHandlers) RemoveObject(r *http.Request, args *RemoveObjectArgs,
|
||||
return toJSONError(errInvalidArgument)
|
||||
}
|
||||
|
||||
reply.UIVersion = browser.UIVersion
|
||||
if isRemoteCallRequired(context.Background(), args.BucketName, objectAPI) {
|
||||
sr, err := globalDNSConfig.Get(args.BucketName)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
return toJSONError(BucketNotFound{
|
||||
Bucket: args.BucketName,
|
||||
}, args.BucketName)
|
||||
}
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
core, err := getRemoteInstanceClient(r, getHostFromSrv(sr))
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
objectsCh := make(chan string)
|
||||
|
||||
// Send object names that are needed to be removed to objectsCh
|
||||
go func() {
|
||||
defer close(objectsCh)
|
||||
|
||||
for _, objectName := range args.Objects {
|
||||
objectsCh <- objectName
|
||||
}
|
||||
}()
|
||||
|
||||
for resp := range core.RemoveObjects(args.BucketName, objectsCh) {
|
||||
if resp.Err != nil {
|
||||
return toJSONError(resp.Err, args.BucketName, resp.ObjectName)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var err error
|
||||
next:
|
||||
for _, objectName := range args.Objects {
|
||||
@@ -479,7 +601,7 @@ next:
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.Action(policy.DeleteObjectAction),
|
||||
Action: iampolicy.DeleteObjectAction,
|
||||
BucketName: args.BucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
@@ -488,7 +610,7 @@ next:
|
||||
return toJSONError(errAccessDenied)
|
||||
}
|
||||
|
||||
if err = deleteObject(nil, objectAPI, web.CacheAPI(), args.BucketName, objectName, r); err != nil {
|
||||
if err = deleteObject(context.Background(), objectAPI, web.CacheAPI(), args.BucketName, objectName, r); err != nil {
|
||||
break next
|
||||
}
|
||||
continue
|
||||
@@ -496,7 +618,7 @@ next:
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.Action(policy.DeleteObjectAction),
|
||||
Action: iampolicy.DeleteObjectAction,
|
||||
BucketName: args.BucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
@@ -515,7 +637,7 @@ next:
|
||||
}
|
||||
marker = lo.NextMarker
|
||||
for _, obj := range lo.Objects {
|
||||
err = deleteObject(nil, objectAPI, web.CacheAPI(), args.BucketName, obj.Name, r)
|
||||
err = deleteObject(context.Background(), objectAPI, web.CacheAPI(), args.BucketName, obj.Name, r)
|
||||
if err != nil {
|
||||
break next
|
||||
}
|
||||
@@ -531,7 +653,6 @@ next:
|
||||
return toJSONError(err, args.BucketName, "")
|
||||
}
|
||||
|
||||
reply.UIVersion = browser.UIVersion
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -605,8 +726,7 @@ func (web *webAPIHandlers) SetAuth(r *http.Request, args *SetAuthArgs, reply *Se
|
||||
}
|
||||
|
||||
// If creds are set through ENV disallow changing credentials.
|
||||
// TODO: Multi-user credentials also cannot be changed from browser.
|
||||
if globalIsEnvCreds || globalWORMEnabled || !owner {
|
||||
if globalIsEnvCreds || globalWORMEnabled || !owner || globalEtcdClient != nil {
|
||||
return toJSONError(errChangeCredNotAllowed)
|
||||
}
|
||||
|
||||
@@ -707,18 +827,13 @@ func (web *webAPIHandlers) CreateURLToken(r *http.Request, args *WebGenericArgs,
|
||||
func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "WebUpload")
|
||||
|
||||
defer logger.AuditLog(w, r, "WebUpload")
|
||||
defer logger.AuditLog(w, r, "WebUpload", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := web.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeWebErrorResponse(w, errServerNotInitialized)
|
||||
return
|
||||
}
|
||||
|
||||
putObject := objectAPI.PutObject
|
||||
if web.CacheAPI() != nil {
|
||||
putObject = web.CacheAPI().PutObject
|
||||
}
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
@@ -747,7 +862,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
if authErr == nil {
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.Action(policy.PutObjectAction),
|
||||
Action: iampolicy.PutObjectAction,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
@@ -757,6 +872,9 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
|
||||
r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
}
|
||||
|
||||
// Require Content-Length to be set in the request
|
||||
size := r.ContentLength
|
||||
@@ -768,13 +886,19 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
// Extract incoming metadata if any.
|
||||
metadata, err := extractMetadata(context.Background(), r)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
reader := r.Body
|
||||
var pReader *PutObjReader
|
||||
var reader io.Reader = r.Body
|
||||
actualSize := size
|
||||
|
||||
hashReader, err := hash.NewReader(reader, size, "", "", actualSize)
|
||||
if err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
if objectAPI.IsCompressionSupported() && isCompressible(r.Header, object) && size > 0 {
|
||||
// Storing the compression metadata.
|
||||
metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV1
|
||||
@@ -800,13 +924,36 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
// Set compression metrics.
|
||||
size = -1 // Since compressed size is un-predictable.
|
||||
reader = pipeReader
|
||||
hashReader, err = hash.NewReader(reader, size, "", "", actualSize)
|
||||
if err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
pReader = NewPutObjReader(hashReader, nil, nil)
|
||||
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if hasServerSideEncryptionHeader(r.Header) && !hasSuffix(object, slashSeparator) { // handle SSE requests
|
||||
rawReader := hashReader
|
||||
var objectEncryptionKey []byte
|
||||
reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
info := ObjectInfo{Size: size}
|
||||
hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", size) // do not try to verify encrypted content
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
|
||||
}
|
||||
}
|
||||
|
||||
hashReader, err := hash.NewReader(reader, size, "", "", actualSize)
|
||||
if err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
// Ensure that metadata does not contain sensitive information
|
||||
crypto.RemoveSensitiveEntries(metadata)
|
||||
|
||||
var opts ObjectOptions
|
||||
// Deny if WORM is enabled
|
||||
if globalWORMEnabled {
|
||||
@@ -816,11 +963,26 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
objInfo, err := putObject(ctx, bucket, object, NewPutObjReader(hashReader, nil, nil), metadata, opts)
|
||||
putObject := objectAPI.PutObject
|
||||
if !hasServerSideEncryptionHeader(r.Header) && web.CacheAPI() != nil {
|
||||
putObject = web.CacheAPI().PutObject
|
||||
}
|
||||
objInfo, err := putObject(context.Background(), bucket, object, pReader, metadata, opts)
|
||||
if err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if crypto.IsEncrypted(objInfo.UserDefined) {
|
||||
switch {
|
||||
case crypto.S3.IsEncrypted(objInfo.UserDefined):
|
||||
w.Header().Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
case crypto.SSEC.IsRequested(r.Header):
|
||||
w.Header().Set(crypto.SSECAlgorithm, r.Header.Get(crypto.SSECAlgorithm))
|
||||
w.Header().Set(crypto.SSECKeyMD5, r.Header.Get(crypto.SSECKeyMD5))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get host and port from Request.RemoteAddr.
|
||||
host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
|
||||
@@ -845,9 +1007,8 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
|
||||
func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "WebDownload")
|
||||
|
||||
defer logger.AuditLog(w, r, "WebDownload")
|
||||
defer logger.AuditLog(w, r, "WebDownload", mustGetClaimsFromToken(r))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
objectAPI := web.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeWebErrorResponse(w, errServerNotInitialized)
|
||||
@@ -883,7 +1044,7 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
|
||||
if authErr == nil {
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.Action(policy.GetObjectAction),
|
||||
Action: iampolicy.GetObjectAction,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
@@ -894,101 +1055,67 @@ func (web *webAPIHandlers) Download(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
opts := ObjectOptions{}
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
getObject := objectAPI.GetObject
|
||||
getObjectNInfo := objectAPI.GetObjectNInfo
|
||||
if web.CacheAPI() != nil {
|
||||
getObjectInfo = web.CacheAPI().GetObjectInfo
|
||||
getObject = web.CacheAPI().GetObject
|
||||
getObjectNInfo = web.CacheAPI().GetObjectNInfo
|
||||
}
|
||||
objInfo, err := getObjectInfo(ctx, bucket, object, opts)
|
||||
|
||||
var opts ObjectOptions
|
||||
gr, err := getObjectNInfo(ctx, bucket, object, nil, r.Header, readLock, opts)
|
||||
if err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
length := objInfo.Size
|
||||
var actualSize int64
|
||||
if objInfo.IsCompressed() {
|
||||
// Read the decompressed size from the meta.json.
|
||||
actualSize = objInfo.GetActualSize()
|
||||
if actualSize < 0 {
|
||||
return
|
||||
}
|
||||
}
|
||||
defer gr.Close()
|
||||
|
||||
objInfo := gr.ObjInfo
|
||||
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if _, err = DecryptObjectInfo(&objInfo, r.Header); err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Set encryption response headers
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if crypto.IsEncrypted(objInfo.UserDefined) {
|
||||
length, _ = objInfo.DecryptedSize()
|
||||
switch {
|
||||
case crypto.S3.IsEncrypted(objInfo.UserDefined):
|
||||
w.Header().Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
|
||||
w.Header().Set(crypto.SSECAlgorithm, r.Header.Get(crypto.SSECAlgorithm))
|
||||
w.Header().Set(crypto.SSECKeyMD5, r.Header.Get(crypto.SSECKeyMD5))
|
||||
}
|
||||
}
|
||||
}
|
||||
var startOffset int64
|
||||
var writer io.Writer
|
||||
if objInfo.IsCompressed() {
|
||||
// The decompress metrics are set.
|
||||
snappyStartOffset := 0
|
||||
snappyLength := actualSize
|
||||
|
||||
// Open a pipe for compression
|
||||
// Where compressWriter is actually passed to the getObject
|
||||
decompressReader, compressWriter := io.Pipe()
|
||||
snappyReader := snappy.NewReader(decompressReader)
|
||||
|
||||
// The limit is set to the actual size.
|
||||
responseWriter := ioutil.LimitedWriter(w, int64(snappyStartOffset), snappyLength)
|
||||
wg.Add(1) //For closures.
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
|
||||
// Finally, writes to the client.
|
||||
_, perr := io.Copy(responseWriter, snappyReader)
|
||||
|
||||
// Close the compressWriter if the data is read already.
|
||||
// Closing the pipe, releases the writer passed to the getObject.
|
||||
compressWriter.CloseWithError(perr)
|
||||
}()
|
||||
writer = compressWriter
|
||||
} else {
|
||||
writer = w
|
||||
}
|
||||
if objectAPI.IsEncryptionSupported() && crypto.S3.IsEncrypted(objInfo.UserDefined) {
|
||||
// Response writer should be limited early on for decryption upto required length,
|
||||
// additionally also skipping mod(offset)64KiB boundaries.
|
||||
writer = ioutil.LimitedWriter(writer, startOffset%(64*1024), length)
|
||||
|
||||
writer, startOffset, length, err = DecryptBlocksRequest(writer, r, bucket, object, startOffset, length, objInfo, false)
|
||||
if err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
}
|
||||
|
||||
httpWriter := ioutil.WriteOnClose(writer)
|
||||
|
||||
// Add content disposition.
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(object)))
|
||||
|
||||
if err = getObject(ctx, bucket, object, 0, -1, httpWriter, "", opts); err != nil {
|
||||
httpWriter.Close()
|
||||
if objInfo.IsCompressed() {
|
||||
wg.Wait()
|
||||
}
|
||||
/// No need to print error, response writer already written to.
|
||||
if err = setObjectHeaders(w, objInfo, nil); err != nil {
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Add content disposition.
|
||||
w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", path.Base(objInfo.Name)))
|
||||
|
||||
setHeadGetRespHeaders(w, r.URL.Query())
|
||||
|
||||
httpWriter := ioutil.WriteOnClose(w)
|
||||
|
||||
// Write object content to response body
|
||||
if _, err = io.Copy(httpWriter, gr); err != nil {
|
||||
if !httpWriter.HasWritten() { // write error response only if no data or headers has been written to client yet
|
||||
writeWebErrorResponse(w, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if err = httpWriter.Close(); err != nil {
|
||||
if !httpWriter.HasWritten() { // write error response only if no data has been written to client yet
|
||||
if !httpWriter.HasWritten() { // write error response only if no data or headers has been written to client yet
|
||||
writeWebErrorResponse(w, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if objInfo.IsCompressed() {
|
||||
// Wait for decompression go-routine to retire.
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// Get host and port from Request.RemoteAddr.
|
||||
host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
|
||||
@@ -1027,7 +1154,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
ctx := newContext(r, w, "WebDownloadZip")
|
||||
defer logger.AuditLog(w, r, "WebDownloadZip")
|
||||
defer logger.AuditLog(w, r, "WebDownloadZip", mustGetClaimsFromToken(r))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
objectAPI := web.ObjectAPI()
|
||||
@@ -1074,7 +1201,7 @@ func (web *webAPIHandlers) DownloadZip(w http.ResponseWriter, r *http.Request) {
|
||||
for _, object := range args.Objects {
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: claims.Subject,
|
||||
Action: iampolicy.Action(policy.GetObjectAction),
|
||||
Action: iampolicy.GetObjectAction,
|
||||
BucketName: args.BucketName,
|
||||
ConditionValues: getConditionValues(r, ""),
|
||||
IsOwner: owner,
|
||||
@@ -1272,18 +1399,48 @@ func (web *webAPIHandlers) GetBucketPolicy(r *http.Request, args *GetBucketPolic
|
||||
return toJSONError(errAccessDenied)
|
||||
}
|
||||
|
||||
bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketPolicyNotFound); !ok {
|
||||
var policyInfo = &miniogopolicy.BucketAccessPolicy{Version: "2012-10-17"}
|
||||
if isRemoteCallRequired(context.Background(), args.BucketName, objectAPI) {
|
||||
sr, err := globalDNSConfig.Get(args.BucketName)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
return toJSONError(BucketNotFound{
|
||||
Bucket: args.BucketName,
|
||||
}, args.BucketName)
|
||||
}
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
return err
|
||||
}
|
||||
client, rerr := getRemoteInstanceClient(r, getHostFromSrv(sr))
|
||||
if rerr != nil {
|
||||
return toJSONError(rerr, args.BucketName)
|
||||
}
|
||||
policyStr, err := client.GetBucketPolicy(args.BucketName)
|
||||
if err != nil {
|
||||
return toJSONError(rerr, args.BucketName)
|
||||
}
|
||||
bucketPolicy, err := policy.ParseConfig(strings.NewReader(policyStr), args.BucketName)
|
||||
if err != nil {
|
||||
return toJSONError(rerr, args.BucketName)
|
||||
}
|
||||
policyInfo, err = PolicyToBucketAccessPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
} else {
|
||||
bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketPolicyNotFound); !ok {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
policyInfo, err := PolicyToBucketAccessPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
policyInfo, err = PolicyToBucketAccessPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
}
|
||||
|
||||
reply.UIVersion = browser.UIVersion
|
||||
@@ -1325,17 +1482,42 @@ func (web *webAPIHandlers) ListAllBucketPolicies(r *http.Request, args *ListAllB
|
||||
return toJSONError(errAccessDenied)
|
||||
}
|
||||
|
||||
bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketPolicyNotFound); !ok {
|
||||
var policyInfo = new(miniogopolicy.BucketAccessPolicy)
|
||||
if isRemoteCallRequired(context.Background(), args.BucketName, objectAPI) {
|
||||
sr, err := globalDNSConfig.Get(args.BucketName)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
return toJSONError(BucketNotFound{
|
||||
Bucket: args.BucketName,
|
||||
}, args.BucketName)
|
||||
}
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
core, rerr := getRemoteInstanceClient(r, getHostFromSrv(sr))
|
||||
if rerr != nil {
|
||||
return toJSONError(rerr, args.BucketName)
|
||||
}
|
||||
var policyStr string
|
||||
policyStr, err = core.Client.GetBucketPolicy(args.BucketName)
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
if policyStr != "" {
|
||||
if err = json.Unmarshal([]byte(policyStr), policyInfo); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
bucketPolicy, err := objectAPI.GetBucketPolicy(context.Background(), args.BucketName)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketPolicyNotFound); !ok {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
}
|
||||
policyInfo, err = PolicyToBucketAccessPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
}
|
||||
|
||||
policyInfo, err := PolicyToBucketAccessPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
reply.UIVersion = browser.UIVersion
|
||||
@@ -1385,43 +1567,94 @@ func (web *webAPIHandlers) SetBucketPolicy(r *http.Request, args *SetBucketPolic
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
bucketPolicy, err := objectAPI.GetBucketPolicy(ctx, args.BucketName)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketPolicyNotFound); !ok {
|
||||
if isRemoteCallRequired(context.Background(), args.BucketName, objectAPI) {
|
||||
sr, err := globalDNSConfig.Get(args.BucketName)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
return toJSONError(BucketNotFound{
|
||||
Bucket: args.BucketName,
|
||||
}, args.BucketName)
|
||||
}
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
}
|
||||
core, rerr := getRemoteInstanceClient(r, getHostFromSrv(sr))
|
||||
if rerr != nil {
|
||||
return toJSONError(rerr, args.BucketName)
|
||||
}
|
||||
var policyStr string
|
||||
// Use the abstracted API instead of core, such that
|
||||
// NoSuchBucketPolicy errors are automatically handled.
|
||||
policyStr, err = core.Client.GetBucketPolicy(args.BucketName)
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
var policyInfo = &miniogopolicy.BucketAccessPolicy{Version: "2012-10-17"}
|
||||
if policyStr != "" {
|
||||
if err = json.Unmarshal([]byte(policyStr), policyInfo); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
}
|
||||
|
||||
policyInfo, err := PolicyToBucketAccessPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
policyInfo.Statements = miniogopolicy.SetPolicy(policyInfo.Statements, policyType, args.BucketName, args.Prefix)
|
||||
if len(policyInfo.Statements) == 0 {
|
||||
if err = core.SetBucketPolicy(args.BucketName, ""); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
policyInfo.Statements = miniogopolicy.SetPolicy(policyInfo.Statements, policyType, args.BucketName, args.Prefix)
|
||||
|
||||
if len(policyInfo.Statements) == 0 {
|
||||
if err = objectAPI.DeleteBucketPolicy(ctx, args.BucketName); err != nil {
|
||||
bucketPolicy, err := BucketAccessPolicyToPolicy(policyInfo)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
globalPolicySys.Remove(args.BucketName)
|
||||
return nil
|
||||
}
|
||||
policyData, err := json.Marshal(bucketPolicy)
|
||||
if err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
bucketPolicy, err = BucketAccessPolicyToPolicy(policyInfo)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
if err = core.SetBucketPolicy(args.BucketName, string(policyData)); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
// Parse validate and save bucket policy.
|
||||
if err := objectAPI.SetBucketPolicy(ctx, args.BucketName, bucketPolicy); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
} else {
|
||||
bucketPolicy, err := objectAPI.GetBucketPolicy(ctx, args.BucketName)
|
||||
if err != nil {
|
||||
if _, ok := err.(BucketPolicyNotFound); !ok {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
}
|
||||
policyInfo, err := PolicyToBucketAccessPolicy(bucketPolicy)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
globalPolicySys.Set(args.BucketName, *bucketPolicy)
|
||||
globalNotificationSys.SetBucketPolicy(ctx, args.BucketName, bucketPolicy)
|
||||
policyInfo.Statements = miniogopolicy.SetPolicy(policyInfo.Statements, policyType, args.BucketName, args.Prefix)
|
||||
if len(policyInfo.Statements) == 0 {
|
||||
if err = objectAPI.DeleteBucketPolicy(ctx, args.BucketName); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
globalPolicySys.Remove(args.BucketName)
|
||||
return nil
|
||||
}
|
||||
|
||||
bucketPolicy, err = BucketAccessPolicyToPolicy(policyInfo)
|
||||
if err != nil {
|
||||
// This should not happen.
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
// Parse validate and save bucket policy.
|
||||
if err := objectAPI.SetBucketPolicy(ctx, args.BucketName, bucketPolicy); err != nil {
|
||||
return toJSONError(err, args.BucketName)
|
||||
}
|
||||
|
||||
globalPolicySys.Set(args.BucketName, *bucketPolicy)
|
||||
globalNotificationSys.SetBucketPolicy(ctx, args.BucketName, bucketPolicy)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -539,8 +539,8 @@ func testListObjectsWebHandler(obj ObjectLayer, instanceType string, t TestErrHa
|
||||
Statements: []policy.Statement{policy.NewStatement(
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.GetObjectAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "*")),
|
||||
policy.NewActionSet(policy.ListBucketAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
condition.NewFunctions(),
|
||||
)},
|
||||
}
|
||||
|
||||
+1
-1
@@ -89,7 +89,7 @@ func registerWebRouter(router *mux.Router) error {
|
||||
|
||||
// These methods use short-expiry tokens in the URLs. These tokens may unintentionally
|
||||
// be logged, so a new one must be generated for each request.
|
||||
webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(web.Download)
|
||||
webBrowserRouter.Methods("GET").Path("/download/{bucket}/{object:.+}").Queries("token", "{token:.*}").HandlerFunc(httpTraceHdrs(web.Download))
|
||||
webBrowserRouter.Methods("POST").Path("/zip").Queries("token", "{token:.*}").HandlerFunc(httpTraceHdrs(web.DownloadZip))
|
||||
|
||||
// Add compression for assets.
|
||||
|
||||
@@ -516,6 +516,11 @@ func (s *xlSets) IsNotificationSupported() bool {
|
||||
return s.getHashedSet("").IsNotificationSupported()
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (s *xlSets) IsListenBucketSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
func (s *xlSets) IsEncryptionSupported() bool {
|
||||
return s.getHashedSet("").IsEncryptionSupported()
|
||||
|
||||
@@ -294,6 +294,11 @@ func (xl xlObjects) IsNotificationSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (xl xlObjects) IsListenBucketSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
func (xl xlObjects) IsEncryptionSupported() bool {
|
||||
return true
|
||||
|
||||
@@ -130,15 +130,22 @@ func (xl xlObjects) listObjects(ctx context.Context, bucket, prefix, marker, del
|
||||
xl.listPool.Set(params, walkResultCh, endWalkCh)
|
||||
}
|
||||
|
||||
result := ListObjectsInfo{IsTruncated: !eof}
|
||||
result := ListObjectsInfo{}
|
||||
for _, objInfo := range objInfos {
|
||||
result.NextMarker = objInfo.Name
|
||||
if objInfo.IsDir && delimiter == slashSeparator {
|
||||
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
||||
continue
|
||||
}
|
||||
result.Objects = append(result.Objects, objInfo)
|
||||
}
|
||||
|
||||
if !eof {
|
||||
result.IsTruncated = true
|
||||
if len(objInfos) > 0 {
|
||||
result.NextMarker = objInfos[len(objInfos)-1].Name
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
|
||||
+22
-14
@@ -34,6 +34,14 @@ func (xl xlObjects) getUploadIDDir(bucket, object, uploadID string) string {
|
||||
return pathJoin(xl.getMultipartSHADir(bucket, object), uploadID)
|
||||
}
|
||||
|
||||
// getUploadIDLockPath returns the name of the Lock in the form of
|
||||
// bucket/object/uploadID. For locking, the path bucket/object/uploadID
|
||||
// is locked instead of multipart-sha256-Dir/uploadID as it is more
|
||||
// readable in the list-locks output which helps in debugging.
|
||||
func (xl xlObjects) getUploadIDLockPath(bucket, object, uploadID string) string {
|
||||
return pathJoin(bucket, object, uploadID)
|
||||
}
|
||||
|
||||
func (xl xlObjects) getMultipartSHADir(bucket, object string) string {
|
||||
return getSHA256Hash([]byte(pathJoin(bucket, object)))
|
||||
}
|
||||
@@ -290,9 +298,10 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
var partsMetadata []xlMetaV1
|
||||
var errs []error
|
||||
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
|
||||
uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
|
||||
|
||||
// pre-check upload id lock.
|
||||
preUploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDPath)
|
||||
preUploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
|
||||
if err := preUploadIDLock.GetRLock(globalOperationTimeout); err != nil {
|
||||
return pi, err
|
||||
}
|
||||
@@ -341,8 +350,10 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
|
||||
// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
|
||||
defer xl.deleteObject(ctx, minioMetaTmpBucket, tmpPart, writeQuorum, false)
|
||||
if data.Size() > 0 || data.Size() == -1 {
|
||||
if pErr := xl.prepareFile(ctx, minioMetaTmpBucket, tmpPartPath, data.Size(), onlineDisks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum); pErr != nil {
|
||||
|
||||
if data.Size() >= 0 {
|
||||
if pErr := xl.prepareFile(ctx, minioMetaTmpBucket, tmpPartPath, data.Size(),
|
||||
onlineDisks, xlMeta.Erasure.BlockSize, xlMeta.Erasure.DataBlocks, writeQuorum); pErr != nil {
|
||||
return pi, toObjectErr(pErr, bucket, object)
|
||||
|
||||
}
|
||||
@@ -369,6 +380,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
if len(buffer) > int(xlMeta.Erasure.BlockSize) {
|
||||
buffer = buffer[:xlMeta.Erasure.BlockSize]
|
||||
}
|
||||
|
||||
writers := make([]*bitrotWriter, len(onlineDisks))
|
||||
for i, disk := range onlineDisks {
|
||||
if disk == nil {
|
||||
@@ -376,6 +388,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
}
|
||||
writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, DefaultBitrotAlgorithm)
|
||||
}
|
||||
|
||||
n, err := erasure.Encode(ctx, data, writers, buffer, erasure.dataBlocks+1)
|
||||
if err != nil {
|
||||
return pi, toObjectErr(err, bucket, object)
|
||||
@@ -394,7 +407,7 @@ func (xl xlObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
}
|
||||
|
||||
// post-upload check (write) lock
|
||||
postUploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDPath)
|
||||
postUploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
|
||||
if err = postUploadIDLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return pi, err
|
||||
}
|
||||
@@ -487,7 +500,7 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI
|
||||
// Hold lock so that there is no competing
|
||||
// abort-multipart-upload or complete-multipart-upload.
|
||||
uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket,
|
||||
xl.getUploadIDDir(bucket, object, uploadID))
|
||||
xl.getUploadIDLockPath(bucket, object, uploadID))
|
||||
if err := uploadIDLock.GetLock(globalListingTimeout); err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -575,6 +588,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
defer destLock.Unlock()
|
||||
|
||||
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
|
||||
uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
|
||||
|
||||
// Hold lock so that
|
||||
//
|
||||
@@ -582,7 +596,7 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
//
|
||||
// 2) no one does a parallel complete-multipart-upload on this
|
||||
// multipart upload
|
||||
uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDPath)
|
||||
uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
|
||||
if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
@@ -674,13 +688,6 @@ func (xl xlObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
}
|
||||
}
|
||||
|
||||
// Last part could have been uploaded as 0bytes, do not need
|
||||
// to save it in final `xl.json`.
|
||||
if (i == len(parts)-1) && currentXLMeta.Parts[partIdx].Size == 0 {
|
||||
xlMeta.Parts = xlMeta.Parts[:i] // Skip the part.
|
||||
continue
|
||||
}
|
||||
|
||||
// Save for total object size.
|
||||
objectSize += currentXLMeta.Parts[partIdx].Size
|
||||
|
||||
@@ -788,9 +795,10 @@ func (xl xlObjects) AbortMultipartUpload(ctx context.Context, bucket, object, up
|
||||
}
|
||||
// Construct uploadIDPath.
|
||||
uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
|
||||
uploadIDLockPath := xl.getUploadIDLockPath(bucket, object, uploadID)
|
||||
// Hold lock so that there is no competing
|
||||
// complete-multipart-upload or put-object-part.
|
||||
uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDPath)
|
||||
uploadIDLock := xl.nsMutex.NewNSLock(minioMetaMultipartBucket, uploadIDLockPath)
|
||||
if err := uploadIDLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+37
-15
@@ -450,12 +450,17 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, op
|
||||
func (xl xlObjects) isObjectCorrupted(metaArr []xlMetaV1, errs []error) (validMeta xlMetaV1, ok bool) {
|
||||
// We can consider an object data not reliable
|
||||
// when xl.json is not found in read quorum disks.
|
||||
var notFoundXLJSON int
|
||||
var notFoundXLJSON, corruptedXLJSON int
|
||||
for _, readErr := range errs {
|
||||
if readErr == errFileNotFound {
|
||||
notFoundXLJSON++
|
||||
}
|
||||
}
|
||||
for _, readErr := range errs {
|
||||
if readErr == errCorruptedFormat {
|
||||
corruptedXLJSON++
|
||||
}
|
||||
}
|
||||
|
||||
for _, m := range metaArr {
|
||||
if !m.IsValid() {
|
||||
@@ -466,24 +471,35 @@ func (xl xlObjects) isObjectCorrupted(metaArr []xlMetaV1, errs []error) (validMe
|
||||
}
|
||||
|
||||
// Return if the object is indeed corrupted.
|
||||
return validMeta, len(xl.getDisks())-notFoundXLJSON < validMeta.Erasure.DataBlocks
|
||||
return validMeta, len(xl.getDisks())-notFoundXLJSON < validMeta.Erasure.DataBlocks || len(xl.getDisks()) == corruptedXLJSON
|
||||
}
|
||||
|
||||
const xlCorruptedSuffix = ".CORRUPTED"
|
||||
|
||||
// Renames the corrupted object and makes it visible.
|
||||
func renameCorruptedObject(ctx context.Context, bucket, object string, validMeta xlMetaV1, disks []StorageAPI, errs []error) {
|
||||
func (xl xlObjects) renameCorruptedObject(ctx context.Context, bucket, object string, validMeta xlMetaV1, disks []StorageAPI, errs []error) {
|
||||
// if errs returned are corrupted
|
||||
if validMeta.Erasure.DataBlocks == 0 {
|
||||
validMeta = newXLMetaV1(object, len(disks)/2, len(disks)/2)
|
||||
}
|
||||
writeQuorum := validMeta.Erasure.DataBlocks + 1
|
||||
|
||||
// Move all existing objects into corrupted suffix.
|
||||
rename(ctx, disks, bucket, object, bucket, object+xlCorruptedSuffix, true, writeQuorum, []error{errFileNotFound})
|
||||
oldObj := mustGetUUID()
|
||||
|
||||
rename(ctx, disks, bucket, object, minioMetaTmpBucket, oldObj, true, writeQuorum, []error{errFileNotFound})
|
||||
|
||||
// Delete temporary object in the event of failure.
|
||||
// If PutObject succeeded there would be no temporary
|
||||
// object to delete.
|
||||
defer xl.deleteObject(ctx, minioMetaTmpBucket, oldObj, writeQuorum, false)
|
||||
|
||||
tempObj := mustGetUUID()
|
||||
|
||||
// Get all the disks which do not have the file.
|
||||
var cdisks = make([]StorageAPI, len(disks))
|
||||
for i, merr := range errs {
|
||||
if merr == errFileNotFound {
|
||||
if merr == errFileNotFound || merr == errCorruptedFormat {
|
||||
cdisks[i] = disks[i]
|
||||
}
|
||||
}
|
||||
@@ -497,18 +513,24 @@ func renameCorruptedObject(ctx context.Context, bucket, object string, validMeta
|
||||
disk.AppendFile(minioMetaTmpBucket, pathJoin(tempObj, "part.1"), []byte{})
|
||||
|
||||
// Write algorithm hash for empty part file.
|
||||
alg := validMeta.Erasure.Checksums[0].Algorithm.New()
|
||||
alg.Write([]byte{})
|
||||
var algorithm = DefaultBitrotAlgorithm
|
||||
h := algorithm.New()
|
||||
h.Write([]byte{})
|
||||
|
||||
// Update the checksums and part info.
|
||||
validMeta.Erasure.Checksums[0] = ChecksumInfo{
|
||||
Name: validMeta.Erasure.Checksums[0].Name,
|
||||
Algorithm: validMeta.Erasure.Checksums[0].Algorithm,
|
||||
Hash: alg.Sum(nil),
|
||||
validMeta.Erasure.Checksums = []ChecksumInfo{
|
||||
{
|
||||
Name: "part.1",
|
||||
Algorithm: algorithm,
|
||||
Hash: h.Sum(nil),
|
||||
},
|
||||
}
|
||||
validMeta.Parts[0] = objectPartInfo{
|
||||
Number: 1,
|
||||
Name: "part.1",
|
||||
|
||||
validMeta.Parts = []objectPartInfo{
|
||||
{
|
||||
Number: 1,
|
||||
Name: "part.1",
|
||||
},
|
||||
}
|
||||
|
||||
// Write the `xl.json` with the newly calculated metadata.
|
||||
@@ -530,7 +552,7 @@ func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (o
|
||||
// Having read quorum means we have xl.json in at least N/2 disks.
|
||||
if !strings.HasSuffix(object, xlCorruptedSuffix) {
|
||||
if validMeta, ok := xl.isObjectCorrupted(metaArr, errs); ok {
|
||||
renameCorruptedObject(ctx, bucket, object, validMeta, disks, errs)
|
||||
xl.renameCorruptedObject(ctx, bucket, object, validMeta, disks, errs)
|
||||
// Return err file not found since we renamed now the corrupted object
|
||||
return objInfo, errFileNotFound
|
||||
}
|
||||
|
||||
@@ -87,8 +87,8 @@ We will enable bucket event notification to trigger whenever a JPEG image is upl
|
||||
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc events add myminio/images arn:minio:sqs::1:amqp --suffix .jpg
|
||||
mc events list myminio/images
|
||||
mc event add myminio/images arn:minio:sqs::1:amqp --suffix .jpg
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:amqp s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -199,8 +199,8 @@ We will enable bucket event notification to trigger whenever a JPEG image is upl
|
||||
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc events add myminio/images arn:minio:sqs::1:mqtt --suffix .jpg
|
||||
mc events list myminio/images
|
||||
mc event add myminio/images arn:minio:sqs::1:mqtt --suffix .jpg
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:amqp s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -314,8 +314,8 @@ With the `mc` tool, the configuration is very simple to add. Let us say that the
|
||||
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc events add myminio/images arn:minio:sqs::1:elasticsearch --suffix .jpg
|
||||
mc events list myminio/images
|
||||
mc event add myminio/images arn:minio:sqs::1:elasticsearch --suffix .jpg
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:elasticsearch s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -460,8 +460,8 @@ With the `mc` tool, the configuration is very simple to add. Let us say that the
|
||||
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc events add myminio/images arn:minio:sqs::1:redis --suffix .jpg
|
||||
mc events list myminio/images
|
||||
mc event add myminio/images arn:minio:sqs::1:redis --suffix .jpg
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:redis s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -563,8 +563,8 @@ We will enable bucket event notification to trigger whenever a JPEG image is upl
|
||||
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc events add myminio/images arn:minio:sqs::1:nats --suffix .jpg
|
||||
mc events list myminio/images
|
||||
mc event add myminio/images arn:minio:sqs::1:nats --suffix .jpg
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:nats s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -745,10 +745,10 @@ With the `mc` tool, the configuration is very simple to add. Let us say that the
|
||||
# Create bucket named `images` in myminio
|
||||
mc mb myminio/images
|
||||
# Add notification configuration on the `images` bucket using the MySQL ARN. The --suffix argument filters events.
|
||||
mc events add myminio/images arn:minio:sqs::1:postgresql --suffix .jpg
|
||||
mc event add myminio/images arn:minio:sqs::1:postgresql --suffix .jpg
|
||||
# Print out the notification configuration on the `images` bucket.
|
||||
mc events list myminio/images
|
||||
mc events list myminio/images
|
||||
mc event list myminio/images
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:postgresql s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -844,9 +844,9 @@ With the `mc` tool, the configuration is very simple to add. Let us say that the
|
||||
# Create bucket named `images` in myminio
|
||||
mc mb myminio/images
|
||||
# Add notification configuration on the `images` bucket using the MySQL ARN. The --suffix argument filters events.
|
||||
mc events add myminio/images arn:minio:sqs::1:postgresql --suffix .jpg
|
||||
mc event add myminio/images arn:minio:sqs::1:postgresql --suffix .jpg
|
||||
# Print out the notification configuration on the `images` bucket.
|
||||
mc events list myminio/images
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:postgresql s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -908,8 +908,8 @@ We will enable bucket event notification to trigger whenever a JPEG image is upl
|
||||
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc events add myminio/images arn:minio:sqs::1:kafka --suffix .jpg
|
||||
mc events list myminio/images
|
||||
mc event add myminio/images arn:minio:sqs::1:kafka --suffix .jpg
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:kafka s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
@@ -965,13 +965,13 @@ We will enable bucket event notification to trigger whenever a JPEG image is upl
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc mb myminio/images-thumbnail
|
||||
mc events add myminio/images arn:minio:sqs::1:webhook --events put --suffix .jpg
|
||||
mc event add myminio/images arn:minio:sqs::1:webhook --events put --suffix .jpg
|
||||
```
|
||||
|
||||
Check if event notification is successfully configured by
|
||||
|
||||
```
|
||||
mc events list myminio/images
|
||||
mc event list myminio/images
|
||||
```
|
||||
|
||||
You should get a response like this
|
||||
@@ -1055,8 +1055,8 @@ We will enable bucket event notification to trigger whenever a JPEG image is upl
|
||||
|
||||
```
|
||||
mc mb myminio/images
|
||||
mc events add myminio/images arn:minio:sqs::1:nsq --suffix .jpg
|
||||
mc events list myminio/images
|
||||
mc event add myminio/images arn:minio:sqs::1:nsq --suffix .jpg
|
||||
mc event list myminio/images
|
||||
arn:minio:sqs::1:nsq s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg”
|
||||
```
|
||||
|
||||
|
||||
+14
-3
@@ -132,9 +132,9 @@ By default, parity for objects with standard storage class is set to `N/2`, and
|
||||
|
||||
## Environment only settings
|
||||
|
||||
#### Browser
|
||||
### Browser
|
||||
|
||||
Enable or disable access to web UI. By default it is set to `on`. You may override this field with ``MINIO_BROWSER`` environment variable.
|
||||
Enable or disable access to web UI. By default it is set to `on`. You may override this field with `MINIO_BROWSER` environment variable.
|
||||
|
||||
Example:
|
||||
|
||||
@@ -145,7 +145,7 @@ minio server /data
|
||||
|
||||
### Domain
|
||||
|
||||
By default, Minio supports path-style requests that are of the format http://mydomain.com/bucket/object. MINIO_DOMAIN environment variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the matched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html)
|
||||
By default, Minio supports path-style requests that are of the format http://mydomain.com/bucket/object. `MINIO_DOMAIN` environment variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the matched pattern `$1` is used as bucket and the path is used as object. More information on path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html)
|
||||
Example:
|
||||
|
||||
```sh
|
||||
@@ -153,6 +153,17 @@ export MINIO_DOMAIN=mydomain.com
|
||||
minio server /data
|
||||
```
|
||||
|
||||
### Drive Sync
|
||||
|
||||
By default, Minio writes to disk in synchronous mode for all metadata operations. Set `MINIO_DRIVE_SYNC` environment variable to enable synchronous mode for all data operations as well.
|
||||
|
||||
Example:
|
||||
|
||||
```sh
|
||||
export MINIO_DRIVE_SYNC=on
|
||||
minio server /data
|
||||
```
|
||||
|
||||
## Explore Further
|
||||
|
||||
* [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide)
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"version": "31",
|
||||
"version": "33",
|
||||
"credential": {
|
||||
"accessKey": "36J9X8EZI4KEV1G7EHXA",
|
||||
"secretKey": "ECk2uqOoNqvtJIMQ3WYugvmNPL_-zm3WcRqP5vUM",
|
||||
@@ -115,7 +115,6 @@
|
||||
"streaming": {
|
||||
"enable": false,
|
||||
"clusterID": "",
|
||||
"clientID": "",
|
||||
"async": false,
|
||||
"maxPubAcksInflight": 0
|
||||
}
|
||||
|
||||
+49
-30
@@ -1,20 +1,26 @@
|
||||
# Minio GCS Gateway [](https://slack.minio.io)
|
||||
Minio GCS Gateway adds Amazon S3 compatibility to Google Cloud Storage.
|
||||
|
||||
## Run Minio Gateway for GCS
|
||||
### Create service account key for GCS and get the credentials file
|
||||
1. Go to the [API Console Credentials page](https://console.developers.google.com/project/_/apis/credentials).
|
||||
2. Select your project or create a new project. Note down your project ID.
|
||||
3. On the Credentials page, select the __Create credentials__ drop-down, then select __Service account key__.
|
||||
4. From the __Service account__ drop-down, select __New service account__
|
||||
5. Fill up __Service account name__ and __Service account ID__
|
||||
6. For the __Role__, click the select dropdown to choose __Storage__ -> __Storage Admin__ _(Full control of GCS resources)_
|
||||
7. Click the __Create__ button. This will download a credentials file to your desktop. Let's call this credentials.json
|
||||
Minio GCS Gateway allows you to access Google Cloud Storage (GCS) with Amazon S3-compatible APIs
|
||||
|
||||
Note: Alternate ways to setup *Application Default Credentials* is explained [here](https://developers.google.com/identity/protocols/application-default-credentials)
|
||||
- [Run Minio Gateway for GCS](#run-minio-gateway-for-gcs)
|
||||
- [Test Using Minio Browser](#test-using-minio-browser)
|
||||
- [Test Using Minio Client](#test-using-minio-client)
|
||||
|
||||
### Using Docker
|
||||
```
|
||||
## <a name="run-minio-gateway-for-gcs"></a>1. Run Minio Gateway for GCS
|
||||
|
||||
### 1.1 Create a Service Account key for GCS and get the Credentials File
|
||||
1. Navigate to the [API Console Credentials page](https://console.developers.google.com/project/_/apis/credentials).
|
||||
2. Select a project or create a new project. Note the project ID.
|
||||
3. Select the **Create credentials** dropdown on the **Credentials** page, and click **Service account key**.
|
||||
4. Select **New service account** from the **Service account** dropdown.
|
||||
5. Populate the **Service account name** and **Service account ID**.
|
||||
6. Click the dropdown for the **Role** and choose **Storage** > **Storage Admin** *(Full control of GCS resources)*.
|
||||
7. Click the **Create** button to download a credentials file and rename it to `credentials.json`.
|
||||
|
||||
**Note:** For alternate ways to set up *Application Default Credentials*, see [Setting Up Authentication for Server to Server Production Applications](https://developers.google.com/identity/protocols/application-default-credentials).
|
||||
|
||||
### 1.1 Run Minio GCS Gateway Using Docker
|
||||
```sh
|
||||
docker run -p 9000:9000 --name gcs-s3 \
|
||||
-v /path/to/credentials.json:/credentials.json \
|
||||
-e "GOOGLE_APPLICATION_CREDENTIALS=/credentials.json" \
|
||||
@@ -23,47 +29,60 @@ docker run -p 9000:9000 --name gcs-s3 \
|
||||
minio/minio gateway gcs yourprojectid
|
||||
```
|
||||
|
||||
### Using Binary
|
||||
```
|
||||
### 1.2 Run Minio GCS Gateway Using the Minio Binary
|
||||
|
||||
```sh
|
||||
export GOOGLE_APPLICATION_CREDENTIALS=/path/to/credentials.json
|
||||
export MINIO_ACCESS_KEY=minioaccesskey
|
||||
export MINIO_SECRET_KEY=miniosecretkey
|
||||
minio gateway gcs yourprojectid
|
||||
```
|
||||
|
||||
## Test using Minio Browser
|
||||
Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully.
|
||||
## <a name="test-using-minio-browser"></a>2. Test Using Minio Browser
|
||||
|
||||
Minio Gateway comes with an embedded web-based object browser that outputs content to http://127.0.0.1:9000. To test that Minio Gateway is running, open a web browser, navigate to http://127.0.0.1:9000, and ensure that the object browser is displayed.
|
||||
|
||||

|
||||
|
||||
## Test using Minio Client `mc`
|
||||
`mc` provides a modern alternative to UNIX commands such as ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services.
|
||||
## <a name="test-using-minio-client"></a>3. Test Using Minio Client
|
||||
|
||||
### Configure `mc`
|
||||
```
|
||||
Minio Client is a command-line tool called `mc` that provides UNIX-like commands for interacting with the server (e.g. ls, cat, cp, mirror, diff, find, etc.). `mc` supports file systems and Amazon S3-compatible cloud storage services (AWS Signature v2 and v4).
|
||||
|
||||
### 3.1 Configure the Gateway using Minio Client
|
||||
|
||||
Use the following command to configure the gateway:
|
||||
|
||||
```sh
|
||||
mc config host add mygcs http://gateway-ip:9000 minioaccesskey miniosecretkey
|
||||
```
|
||||
|
||||
### List containers on GCS
|
||||
```
|
||||
### 3.2 List Containers on GCS
|
||||
|
||||
Use the following command to list the containers on GCS:
|
||||
|
||||
```sh
|
||||
mc ls mygcs
|
||||
```
|
||||
|
||||
A response similar to this one should be displayed:
|
||||
|
||||
```
|
||||
[2017-02-22 01:50:43 PST] 0B ferenginar/
|
||||
[2017-02-26 21:43:51 PST] 0B my-container/
|
||||
[2017-02-26 22:10:11 PST] 0B test-container1/
|
||||
```
|
||||
|
||||
### Known limitations
|
||||
Gateway inherits the following GCS limitations:
|
||||
### 3.3 Known limitations
|
||||
Minio Gateway has the following limitations when used with GCS:
|
||||
|
||||
- Only read-only or write-only bucket policy supported at bucket level, all other variations will return API Notimplemented error.
|
||||
- _List Multipart Uploads_ and _List Object parts_ always returns empty list. i.e Client will need to remember all the parts that it has uploaded and use it for _Complete Multipart Upload_
|
||||
* It only supports read-only and write-only bucket policies at the bucket level; all other variations will return `API Not implemented`.
|
||||
* The `List Multipart Uploads` and `List Object parts` commands always return empty lists. Therefore, the client must store all of the parts that it has uploaded and use that information when invoking the `_Complete Multipart Upload` command.
|
||||
|
||||
Other limitations:
|
||||
|
||||
- Bucket notification APIs are not supported.
|
||||
* Bucket notification APIs are not supported.
|
||||
|
||||
## Explore Further
|
||||
## <a name="explore-further"></a>4. Explore Further
|
||||
- [`mc` command-line interface](https://docs.minio.io/docs/minio-client-quickstart-guide)
|
||||
- [`aws` command-line interface](https://docs.minio.io/docs/aws-cli-with-minio)
|
||||
- [`minio-go` Go SDK](https://docs.minio.io/docs/golang-client-quickstart-guide)
|
||||
|
||||
|
||||
+15
-3
@@ -1,35 +1,47 @@
|
||||
# Minio NAS Gateway [](https://slack.minio.io)
|
||||
|
||||
Minio Gateway adds Amazon S3 compatibility to NAS storage. You may run multiple minio instances on the same shared NAS volume as a distributed object gateway.
|
||||
|
||||
## Run Minio Gateway for NAS Storage
|
||||
|
||||
### Using Docker
|
||||
|
||||
Please ensure to replace `/shared/nasvol` with actual mount path.
|
||||
|
||||
```
|
||||
docker run -p 9000:9000 --name nas-s3 \
|
||||
-e "MINIO_ACCESS_KEY=minio" \
|
||||
-e "MINIO_SECRET_KEY=minio123" \
|
||||
minio/minio gateway nas /shared/nasvol
|
||||
-v /shared/nasvol:/container/vol \
|
||||
minio/minio gateway nas /container/vol
|
||||
```
|
||||
|
||||
### Using Binary
|
||||
|
||||
```
|
||||
export MINIO_ACCESS_KEY=minioaccesskey
|
||||
export MINIO_SECRET_KEY=miniosecretkey
|
||||
export MINIO_ACCESS_KEY=minio
|
||||
export MINIO_SECRET_KEY=minio123
|
||||
minio gateway nas /shared/nasvol
|
||||
```
|
||||
|
||||
## Test using Minio Browser
|
||||
|
||||
Minio Gateway comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure that your server has started successfully.
|
||||
|
||||

|
||||
|
||||
## Test using Minio Client `mc`
|
||||
|
||||
`mc` provides a modern alternative to UNIX commands such as ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services.
|
||||
|
||||
### Configure `mc`
|
||||
|
||||
```
|
||||
mc config host add mynas http://gateway-ip:9000 access_key secret_key
|
||||
```
|
||||
|
||||
### List buckets on nas
|
||||
|
||||
```
|
||||
mc ls mynas
|
||||
[2017-02-22 01:50:43 PST] 0B ferenginar/
|
||||
|
||||
+101
-12
@@ -1,26 +1,57 @@
|
||||
# KMS Quickstart Guide [](https://slack.minio.io)
|
||||
|
||||
KMS feature allows you to use Vault to generate and manage encryption keys for use by the minio server to encrypt objects. This document explains how to configure Minio with Vault as KMS.
|
||||
Minio uses a key-management-system (KMS) to support SSE-S3. If a client requests SSE-S3, or auto-encryption
|
||||
is enabled, the Minio server encrypts each object with an unique object key which is protected by a master key
|
||||
managed by the KMS. Usually all object keys are protected by a single master key.
|
||||
|
||||
Minio supports two different KMS concepts:
|
||||
- External KMS:
|
||||
Minio can be configured to use an external KMS i.e. [Hashicorp Vault](https://www.vaultproject.io/).
|
||||
An external KMS decouples Minio as storage system from key-management. An external KMS can
|
||||
be managed by a dedicated security team and allows you to grant/deny access to (certain) objects
|
||||
by enabling or disabling the corresponding master keys on demand.
|
||||
|
||||
- Direct KMS master keys:
|
||||
Minio can also be configured to directly use a master key specified by the environment variable `MINIO_SSE_MASTER_KEY`.
|
||||
Direct master keys are useful if the storage backend is not on the same machine as the Minio server, e.g.,
|
||||
if network drives or Minio gateway is used and an external KMS would cause too much management overhead.
|
||||
|
||||
Note: KMS master keys are mainly for testing purposes. It's not recommended to use them for production deployments.
|
||||
Further if the Minio server machine is ever compromised, then the master key must also be treated as compromised.
|
||||
|
||||
**Important:**
|
||||
If multiple Minio servers are configured as [gateways](https://github.com/minio/minio/blob/master/docs/gateway/README.md)
|
||||
pointing to the *same* backend - for example the same NAS storage - then the KMS configuration **must** be the same for
|
||||
all gateways. Otherwise one gateway may not be able to decrypt objects created by another gateway. It is the operators'
|
||||
responsibility to ensure consistency.
|
||||
|
||||
## Get started
|
||||
|
||||
### 1. Prerequisites
|
||||
Install Minio - [Minio Quickstart Guide](https://docs.minio.io/docs/minio-quickstart-guide).
|
||||
|
||||
### 2. Configure Vault
|
||||
Vault as Key Management System requires following to be configured in Vault
|
||||
### 2. Setup a KMS
|
||||
|
||||
- [transit](https://www.vaultproject.io/docs/secrets/transit/index.html) backend configured with a named encryption key-ring
|
||||
- [AppRole](https://www.vaultproject.io/docs/auth/approle.html) based authentication with read/update policy for transit backend. In particular, read and update policy are required for the [Generate Data Key](https://www.vaultproject.io/api/secret/transit/index.html#generate-data-key) endpoint and [Decrypt Data](https://www.vaultproject.io/api/secret/transit/index.html#decrypt-data) endpoint.
|
||||
Either use Hashicorp Vault as external KMS or specify a master key directly depending on your use case.
|
||||
|
||||
#### 2.1 Setup Hashicorp Vault
|
||||
|
||||
Here is a sample quick start for configuring vault with a transit backend and Approle with correct policy
|
||||
#### 2.1 Start Vault server in dev mode
|
||||
|
||||
Minio requires the following Vault setup:
|
||||
- The [transit backend](https://www.vaultproject.io/api/secret/transit/index.html) configured with a named encryption key-ring
|
||||
- [AppRole](https://www.vaultproject.io/docs/auth/approle.html) based authentication with read/update policy for transit backend. In particular, read and update policy are required for the [Generate Data Key](https://www.vaultproject.io/api/secret/transit/index.html#generate-data-key) endpoint and [Decrypt Data](https://www.vaultproject.io/api/secret/transit/index.html#decrypt-data) endpoint.
|
||||
|
||||
**2.1.1 Start Vault server in dev mode**
|
||||
|
||||
In dev mode, Vault server runs in-memory and starts unsealed. Note that running Vault in dev mode is insecure and any data stored in the Vault is lost upon restart.
|
||||
|
||||
```
|
||||
vault server -dev
|
||||
```
|
||||
|
||||
#### 2.2 Set up vault transit backend and create an app role
|
||||
**2.1.2 Set up vault transit backend and create an app role**
|
||||
|
||||
```
|
||||
cat > vaultpolicy.hcl <<EOF
|
||||
path "transit/datakey/plaintext/my-minio-key" {
|
||||
@@ -50,27 +81,85 @@ vault write -f auth/approle/role/my-role/secret-id
|
||||
|
||||
The AppRole ID, AppRole Secret Id, Vault endpoint and Vault key name can now be used to start minio server with Vault as KMS.
|
||||
|
||||
### 3. Environment variables
|
||||
**2.1.3 Vault Environment variables**
|
||||
|
||||
You'll need the Vault endpoint, AppRole ID, AppRole SecretID and encryption key-ring name defined in step 2.2
|
||||
You'll need the Vault endpoint, AppRole ID, AppRole SecretID and encryption key-ring name defined in step 2.1.2
|
||||
|
||||
```sh
|
||||
export MINIO_SSE_VAULT_APPROLE_ID=9b56cc08-8258-45d5-24a3-679876769126
|
||||
export MINIO_SSE_VAULT_APPROLE_SECRET=4e30c52f-13e4-a6f5-0763-d50e8cb4321f
|
||||
export MINIO_SSE_VAULT_ENDPOINT=https://vault-endpoint-ip:8200
|
||||
export MINIO_SSE_VAULT_KEY_NAME=my-minio-key
|
||||
export MINIO_SSE_VAULT_AUTH_TYPE=approle
|
||||
minio server ~/export
|
||||
```
|
||||
|
||||
Optionally set `MINIO_SSE_VAULT_CAPATH` as the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.
|
||||
Optionally, set `MINIO_SSE_VAULT_CAPATH` to a directory of PEM-encoded CA cert files to use mTLS for client-server authentication.
|
||||
|
||||
```
|
||||
export MINIO_SSE_VAULT_CAPATH=/home/user/custom-pems
|
||||
export MINIO_SSE_VAULT_CAPATH=/home/user/custom-certs
|
||||
```
|
||||
|
||||
### 4. Test your setup
|
||||
An additional option is to set `MINIO_SSE_VAULT_NAMESPACE` if AppRole and Transit Secrets engine have been scoped to Vault Namespace
|
||||
|
||||
```
|
||||
export MINIO_SSE_VAULT_NAMESPACE=ns1
|
||||
```
|
||||
|
||||
Note: If [Vault Namespaces](https://learn.hashicorp.com/vault/operations/namespaces) are in use, MINIO_SSE_VAULT_NAMESPACE variable needs to be set before setting approle and transit secrets engine.
|
||||
|
||||
|
||||
|
||||
#### 2.2 Specify a master key
|
||||
|
||||
A KMS master key consists of a master-key ID (CMK) and the 256 bit master key encoded as HEX value separated by a `:`.
|
||||
A KMS master key can be specified directly using:
|
||||
|
||||
```sh
|
||||
export MINIO_SSE_MASTER_KEY=my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574
|
||||
```
|
||||
|
||||
Please use your own master key. A random master key can be generated using e.g. this command on Linux/Mac/BSD* systems:
|
||||
|
||||
```sh
|
||||
head -c 32 /dev/urandom | xxd -c 32 -ps
|
||||
```
|
||||
|
||||
### 3. Test your setup
|
||||
|
||||
To test this setup, start minio server with environment variables set in Step 3, and server is ready to handle SSE-S3 requests.
|
||||
|
||||
### Auto-Encryption
|
||||
|
||||
Minio can also enable auto-encryption **if** a valid KMS configuration is specified and the storage backend supports
|
||||
encrypted objects. Auto-Encryption, if enabled, ensures that all uploaded objects are encrypted using the specified
|
||||
KMS configuration.
|
||||
|
||||
Auto-Encryption is useful especially if the Minio operator wants to ensure that objects are **never** stored in
|
||||
plaintext - for example if sensitive data is stored on public cloud storage.
|
||||
|
||||
To enable auto-encryption set the environment variable to `on`:
|
||||
|
||||
```sh
|
||||
export MINIO_SSE_AUTO_ENCRYPTION=on
|
||||
```
|
||||
|
||||
To verify auto-encryption, use the `mc` command:
|
||||
|
||||
```sh
|
||||
mc cp test.file myminio/crypt/
|
||||
test.file: 5 B / 5 B ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 100.00% 337 B/s 0s
|
||||
mc stat myminio/crypt/test.file
|
||||
Name : test.file
|
||||
...
|
||||
Encrypted :
|
||||
X-Amz-Server-Side-Encryption: AES256
|
||||
```
|
||||
|
||||
Note: Auto-Encryption only affects non-SSE-C requests since objects uploaded using SSE-C are already encrypted
|
||||
and S3 only allows either SSE-S3 or SSE-C but not both for the same object.
|
||||
|
||||
|
||||
# Explore Further
|
||||
|
||||
- [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
|
||||
|
||||
+29
-11
@@ -77,27 +77,45 @@ Setting this environment variable automatically enables audit logging to the HTT
|
||||
{
|
||||
"version": "1",
|
||||
"deploymentid": "1b3002bf-5005-4d9b-853e-64a05008ebb2",
|
||||
"time": "2018-11-02T21:57:58.231480177Z",
|
||||
"time": "2018-11-21T23:16:06.828154172Z",
|
||||
"api": {
|
||||
"name": "ListBuckets",
|
||||
"args": {}
|
||||
"name": "PutObject",
|
||||
"bucket": "my-bucketname",
|
||||
"object": "my-objectname",
|
||||
"status": "OK",
|
||||
"statusCode": 200
|
||||
},
|
||||
"remotehost": "127.0.0.1",
|
||||
"requestID": "15636D7C53428FD4",
|
||||
"userAgent": "Minio (linux; amd64) minio-go/v6.0.8 mc/2018-11-02T21:13:30Z",
|
||||
"requestID": "156946C6C1E7842C",
|
||||
"userAgent": "Minio (linux; amd64) minio-go/v6.0.6",
|
||||
"requestClaims": {
|
||||
"accessKey": "A1YABB5YPX3ZPL4227XJ",
|
||||
"aud": "PoEgXP6uVO45IsENRngDXj5Au5Ya",
|
||||
"azp": "PoEgXP6uVO45IsENRngDXj5Au5Ya",
|
||||
"exp": 1542845766,
|
||||
"iat": 1542842166,
|
||||
"iss": "https://localhost:9443/oauth2/token",
|
||||
"jti": "33527fcc-254f-43d2-a558-4942554b8ff8"
|
||||
},
|
||||
"requestHeader": {
|
||||
"Authorization": "AWS4-HMAC-SHA256 Credential=minio/20181102/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=6db486b42a85b23bffba66d654ce60242a7e92fb27cd4a1756e68082c02cc204",
|
||||
"User-Agent": "Minio (linux; amd64) minio-go/v6.0.8 mc/2018-11-02T21:13:30Z",
|
||||
"X-Amz-Content-Sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
|
||||
"X-Amz-Date": "20181102T215758Z"
|
||||
"Authorization": "AWS4-HMAC-SHA256 Credential=A1YABB5YPX3ZPL4227XJ/20181121/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-security-token,Signature=689d9b8f67b5625ea2f0b8cbb3f777d8839a91d50aa81e6a5555f5a6360c1714",
|
||||
"Content-Length": "184",
|
||||
"Content-Type": "application/octet-stream",
|
||||
"User-Agent": "Minio (linux; amd64) minio-go/v6.0.6",
|
||||
"X-Amz-Content-Sha256": "STREAMING-AWS4-HMAC-SHA256-PAYLOAD",
|
||||
"X-Amz-Date": "20181121T231606Z",
|
||||
"X-Amz-Decoded-Content-Length": "12",
|
||||
"X-Amz-Security-Token": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJBMVlBQkI1WVBYM1pQTDQyMjdYSiIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQyODQ1NzY2LCJpYXQiOjE1NDI4NDIxNjYsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiMzM1MjdmY2MtMjU0Zi00M2QyLWE1NTgtNDk0MjU1NGI4ZmY4In0.KEuAq2cQ3H7dfIB5DVuvcgBXT38mr0gthrIbVRSZcA2OWo8QiH1-DWXj9xYbndgr1p2tiEUsQ49cuszQGEVGMQ"
|
||||
},
|
||||
"responseHeader": {
|
||||
"Accept-Ranges": "bytes",
|
||||
"Content-Security-Policy": "block-all-mixed-content",
|
||||
"Content-Type": "application/xml",
|
||||
"Server": "Minio/DEVELOPMENT.2018-11-02T21-57-15Z (linux; amd64)",
|
||||
"Etag": "",
|
||||
"Server": "Minio/DEVELOPMENT.2018-11-21T23-15-06Z (linux; amd64)",
|
||||
"Vary": "Origin",
|
||||
"X-Amz-Request-Id": "15636D7C53428FD4",
|
||||
"X-Amz-Request-Id": "156946C6C1E7842C",
|
||||
"X-Minio-Deployment-Id": "1b3002bf-5005-4d9b-853e-64a05008ebb2",
|
||||
"X-Xss-Protection": "1; mode=block"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,19 @@
|
||||
# Multi-tenant Minio Deployment Guide [](https://slack.minio.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
# Minio Multi-Tenant Deployment Guide [](https://slack.minio.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
|
||||
## Standalone Deployment
|
||||
To host multiple tenants on a single machine, run one Minio server per tenant with dedicated HTTPS port, config and data directory.
|
||||
This topic provides commands to set up different configurations of hosts, nodes, and drives. The examples provided here can be used as a starting point for other configurations.
|
||||
|
||||
#### Example 1 : Single host, single drive
|
||||
1. [Standalone Deployment](#standalone-deployment)
|
||||
2. [Distributed Deployment](#distributed-deployment)
|
||||
3. [Cloud Scale Deployment](#cloud-scale-deployment)
|
||||
|
||||
## <a name="standalone-deployment"></a>1. Standalone Deployment
|
||||
|
||||
To host multiple tenants on a single machine, run one Minio Server per tenant with a dedicated HTTPS port, configuration, and data directory.
|
||||
|
||||
### 1.1 Host Multiple Tenants on a Single Drive
|
||||
|
||||
Use the following commands to host 3 tenants on a single drive:
|
||||
|
||||
This example hosts 3 tenants on a single drive.
|
||||
```sh
|
||||
minio --config-dir ~/tenant1 server --address :9001 /data/tenant1
|
||||
minio --config-dir ~/tenant2 server --address :9002 /data/tenant2
|
||||
@@ -14,22 +22,25 @@ minio --config-dir ~/tenant3 server --address :9003 /data/tenant3
|
||||
|
||||

|
||||
|
||||
#### Example 2 : Single host, multiple drives (erasure code)
|
||||
### 1.2 Host Multiple Tenants on Multiple Drives (Erasure Code)
|
||||
|
||||
Use the following commands to host 3 tenants on multiple drives:
|
||||
|
||||
This example hosts 3 tenants on multiple drives.
|
||||
```sh
|
||||
minio --config-dir ~/tenant1 server --address :9001 /disk1/data/tenant1 /disk2/data/tenant1 /disk3/data/tenant1 /disk4/data/tenant1
|
||||
minio --config-dir ~/tenant2 server --address :9002 /disk1/data/tenant2 /disk2/data/tenant2 /disk3/data/tenant2 /disk4/data/tenant2
|
||||
minio --config-dir ~/tenant3 server --address :9003 /disk1/data/tenant3 /disk2/data/tenant3 /disk3/data/tenant3 /disk4/data/tenant3
|
||||
```
|
||||
|
||||

|
||||
|
||||
## Distributed Deployment
|
||||
To host multiple tenants in a distributed environment, run several distributed Minio instances concurrently.
|
||||
## <a name="distributed-deployment"></a>2. Distributed Deployment
|
||||
|
||||
#### Example 1 : Multiple host, multiple drives (erasure code)
|
||||
To host multiple tenants in a distributed environment, run several distributed Minio Server instances concurrently.
|
||||
|
||||
This example hosts 3 tenants on a 4 node distributed setup. Execute the following command on all the four nodes.
|
||||
### 2.1 Host Multiple Tenants on Multiple Drives (Erasure Code)
|
||||
|
||||
Use the following commands to host 3 tenants on a 4-node distributed configuration:
|
||||
|
||||
```sh
|
||||
export MINIO_ACCESS_KEY=<TENANT1_ACCESS_KEY>
|
||||
@@ -45,14 +56,12 @@ export MINIO_SECRET_KEY=<TENANT3_SECRET_KEY>
|
||||
minio --config-dir ~/tenant3 server --address :9003 http://192.168.10.11/data/tenant3 http://192.168.10.12/data/tenant3 http://192.168.10.13/data/tenant3 http://192.168.10.14/data/tenant3
|
||||
```
|
||||
|
||||
**Note:** Execute the commands on all 4 nodes.
|
||||
|
||||

|
||||
|
||||
*Note*
|
||||
|
||||
Please remember that defining and exporting credential environment variables, `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY`, is **mandatory** on distributed deployments. It is also required to define and export domain environment variable, `MINIO_DOMAIN`, if domain information is needed to be set.
|
||||
|
||||
## Cloud Scale Deployment
|
||||
|
||||
For large scale multi-tenant Minio deployments, we recommend using one of the popular container orchestration platforms, e.g. Kubernetes, DC/OS or Docker Swarm. Refer [this document](https://docs.minio.io/docs/minio-deployment-quickstart-guide) to get started with Minio on orchestration platforms.
|
||||
**Note**: On distributed systems, credentials must be defined and exported using the `MINIO_ACCESS_KEY` and `MINIO_SECRET_KEY` environment variables. If a domain is required, it must be specified by defining and exporting the `MINIO_DOMAIN` environment variable.
|
||||
|
||||
## <a name="cloud-scale-deployment"></a>Cloud Scale Deployment
|
||||
|
||||
A container orchestration platform (e.g. Kubernetes, DC/OS, or Docker Swarm) is recommended for large-scale, multi-tenant Minio deployments. See the [Minio Deployment Quickstart Guide](https://docs.minio.io/docs/minio-deployment-quickstart-guide) to get started with Minio on orchestration platforms.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user