mirror of
https://github.com/pgsty/minio.git
synced 2026-11-02 05:42:06 +03:00
Compare commits
81 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7881791a91 | |||
| d2f8f8c7ee | |||
| 8c32311b80 | |||
| 9bb88e610e | |||
| d1e41695fe | |||
| 2aeb3fbe86 | |||
| 99b843a64e | |||
| 4f31a9a33b | |||
| 65ddff8899 | |||
| e7c902bbbc | |||
| 5a5895203b | |||
| 7da0336ac8 | |||
| 3be616de3f | |||
| c5bf22fd90 | |||
| 7c9f934875 | |||
| b6f9b24b30 | |||
| 13cb814a0e | |||
| d264d2c899 | |||
| bebaff269c | |||
| 52b159b1db | |||
| 324834e4da | |||
| c2ed1347d9 | |||
| 50f6f9fe58 | |||
| 48cb0ea34b | |||
| 6f7c99a333 | |||
| 3498f5b0ec | |||
| 21d8c0fd13 | |||
| 79b9a9ce46 | |||
| b9b353db4b | |||
| 11a9b317a3 | |||
| 9af7d627ac | |||
| 76d9d54603 | |||
| 4c7c571875 | |||
| 313ba74b09 | |||
| 3e124315c8 | |||
| 78a0fd951e | |||
| 40852801ea | |||
| f6980c4630 | |||
| 8fcc787cba | |||
| 4e6d3c093f | |||
| 61145361fd | |||
| 950b4ad9af | |||
| 6add646130 | |||
| 20e61fb362 | |||
| 18ced1102c | |||
| d6af3c1237 | |||
| 5549a44566 | |||
| e7af31c2ff | |||
| e7971b1d55 | |||
| 26120d7838 | |||
| bef7c01c58 | |||
| 6a8ccc5925 | |||
| d85199e9de | |||
| 8608a84c23 | |||
| b7226f4c82 | |||
| f930ffe9e2 | |||
| b50a245208 | |||
| 45bb11e020 | |||
| b65cf281fd | |||
| f781548b0c | |||
| 25ee8e74f7 | |||
| c975d2cc7e | |||
| ea66528739 | |||
| e1164103d4 | |||
| 83fe70f710 | |||
| 12a6523fb2 | |||
| dba61867e8 | |||
| dd092f6c2b | |||
| 2a810c7da2 | |||
| dd8c2aa5c6 | |||
| 9e3fce441e | |||
| d4265f9a13 | |||
| 2fc024e880 | |||
| eddf468aef | |||
| b0d04b9a81 | |||
| a9de303d8b | |||
| 69bd6df464 | |||
| bfb505aa8e | |||
| d732b1ff9d | |||
| ef517bd0da | |||
| 32d837cf88 |
@@ -34,6 +34,11 @@ matrix:
|
||||
- for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 20m "$d"; done
|
||||
- bash buildscripts/go-coverage.sh
|
||||
|
||||
before_script:
|
||||
# Add an IPv6 config - see the corresponding Travis issue
|
||||
# https://github.com/travis-ci/travis-ci/issues/8361
|
||||
- if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then sudo sh -c 'echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6'; fi
|
||||
|
||||
before_install:
|
||||
- if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then nvm install stable ; fi
|
||||
|
||||
|
||||
@@ -68,6 +68,36 @@ function start_minio_erasure_sets()
|
||||
echo "$minio_pid"
|
||||
}
|
||||
|
||||
function start_minio_dist_erasure_sets_ipv6()
|
||||
{
|
||||
declare -a minio_pids
|
||||
export MINIO_ACCESS_KEY=$ACCESS_KEY
|
||||
export MINIO_SECRET_KEY=$SECRET_KEY
|
||||
"${MINIO[@]}" server --address="[::1]:9000" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9000.log" 2>&1 &
|
||||
minio_pids[0]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9001" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9001.log" 2>&1 &
|
||||
minio_pids[1]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9002" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9002.log" 2>&1 &
|
||||
minio_pids[2]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9003" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9003.log" 2>&1 &
|
||||
minio_pids[3]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9004" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9004.log" 2>&1 &
|
||||
minio_pids[4]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9005" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9005.log" 2>&1 &
|
||||
minio_pids[5]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9006" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9006.log" 2>&1 &
|
||||
minio_pids[6]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9007" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9007.log" 2>&1 &
|
||||
minio_pids[7]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9008" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9008.log" 2>&1 &
|
||||
minio_pids[8]=$!
|
||||
"${MINIO[@]}" server --address="[::1]:9009" "http://[::1]:9000${WORK_DIR}/dist-disk-sets1" "http://[::1]:9001${WORK_DIR}/dist-disk-sets2" "http://[::1]:9002${WORK_DIR}/dist-disk-sets3" "http://[::1]:9003${WORK_DIR}/dist-disk-sets4" "http://[::1]:9004${WORK_DIR}/dist-disk-sets5" "http://[::1]:9005${WORK_DIR}/dist-disk-sets6" "http://[::1]:9006${WORK_DIR}/dist-disk-sets7" "http://[::1]:9007${WORK_DIR}/dist-disk-sets8" "http://[::1]:9008${WORK_DIR}/dist-disk-sets9" "http://[::1]:9009${WORK_DIR}/dist-disk-sets10" "http://[::1]:9000${WORK_DIR}/dist-disk-sets11" "http://[::1]:9001${WORK_DIR}/dist-disk-sets12" "http://[::1]:9002${WORK_DIR}/dist-disk-sets13" "http://[::1]:9003${WORK_DIR}/dist-disk-sets14" "http://[::1]:9004${WORK_DIR}/dist-disk-sets15" "http://[::1]:9005${WORK_DIR}/dist-disk-sets16" "http://[::1]:9006${WORK_DIR}/dist-disk-sets17" "http://[::1]:9007${WORK_DIR}/dist-disk-sets18" "http://[::1]:9008${WORK_DIR}/dist-disk-sets19" "http://[::1]:9009${WORK_DIR}/dist-disk-sets20" >"$WORK_DIR/dist-minio-v6-9009.log" 2>&1 &
|
||||
minio_pids[9]=$!
|
||||
|
||||
sleep 35
|
||||
echo "${minio_pids[@]}"
|
||||
}
|
||||
|
||||
function start_minio_dist_erasure_sets()
|
||||
{
|
||||
declare -a minio_pids
|
||||
@@ -161,6 +191,34 @@ function run_test_erasure_sets() {
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function run_test_dist_erasure_sets_ipv6()
|
||||
{
|
||||
minio_pids=( $(start_minio_dist_erasure_sets_ipv6) )
|
||||
|
||||
export SERVER_ENDPOINT="[::1]:9000"
|
||||
|
||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||
rv=$?
|
||||
|
||||
for pid in "${minio_pids[@]}"; do
|
||||
kill "$pid"
|
||||
done
|
||||
sleep 3
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
for i in $(seq 0 9); do
|
||||
echo "server$i log:"
|
||||
cat "$WORK_DIR/dist-minio-v6-900$i.log"
|
||||
done
|
||||
fi
|
||||
|
||||
for i in $(seq 0 9); do
|
||||
rm -f "$WORK_DIR/dist-minio-v6-900$i.log"
|
||||
done
|
||||
|
||||
return "$rv"
|
||||
}
|
||||
|
||||
function run_test_dist_erasure_sets()
|
||||
{
|
||||
minio_pids=( $(start_minio_dist_erasure_sets) )
|
||||
@@ -237,6 +295,7 @@ function run_test_gateway_s3()
|
||||
{
|
||||
minio_pid="$(start_minio_gateway_s3)"
|
||||
|
||||
export SERVER_ENDPOINT="127.0.0.1:9000"
|
||||
export ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
export SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
|
||||
@@ -279,6 +338,7 @@ function __init__()
|
||||
exit 1
|
||||
fi
|
||||
|
||||
sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
|
||||
chmod a+x "$FUNCTIONAL_TESTS"
|
||||
}
|
||||
|
||||
@@ -319,6 +379,13 @@ function main()
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Distributed Erasure setup as sets with ipv6"
|
||||
if ! run_test_dist_erasure_sets_ipv6; then
|
||||
echo "FAILED"
|
||||
rm -fr "$WORK_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing in Gateway S3 setup"
|
||||
if ! run_test_gateway_s3; then
|
||||
echo "FAILED"
|
||||
|
||||
+10
-10
@@ -57,28 +57,28 @@ type accessControlPolicy struct {
|
||||
func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketACL")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "GetBucketACL", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Allow getBucketACL if policy action is set, since this is a dummy call
|
||||
// we are simply re-purposing the bucketPolicyAction.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -92,7 +92,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
Permission: "FULL_CONTROL",
|
||||
})
|
||||
if err := xml.NewEncoder(w).Encode(acl); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.
|
||||
func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetObjectACL")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "GetObjectACL", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -114,21 +114,21 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Allow getObjectACL if policy action is set, since this is a dummy call
|
||||
// we are simply re-purposing the bucketPolicyAction.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Before proceeding validate if object exists.
|
||||
_, err := objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -142,7 +142,7 @@ func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.
|
||||
Permission: "FULL_CONTROL",
|
||||
})
|
||||
if err := xml.NewEncoder(w).Encode(acl); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+43
-3
@@ -578,7 +578,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
StartTime: nh.startTime,
|
||||
})
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// Client token not specified but a heal sequence exists on a path,
|
||||
@@ -859,7 +859,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
|
||||
|
||||
// Custom IAM policies not allowed for admin user.
|
||||
if accessKey == globalServerConfig.GetCredential().AccessKey {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -867,6 +867,14 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddUser - PUT /minio/admin/v1/add-user?accessKey=<access_key>
|
||||
@@ -898,7 +906,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Custom IAM policies not allowed for admin user.
|
||||
if accessKey == globalServerConfig.GetCredential().AccessKey {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
writeErrorResponseJSON(w, ErrInvalidRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -927,6 +935,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ListCannedPolicies - GET /minio/admin/v1/list-canned-policies
|
||||
@@ -992,6 +1008,14 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// AddCannedPolicy - PUT /minio/admin/v1/add-canned-policy?name=<policy_name>
|
||||
@@ -1049,6 +1073,14 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetUserPolicy - PUT /minio/admin/v1/set-user-policy?accessKey=<access_key>&name=<policy_name>
|
||||
@@ -1088,6 +1120,14 @@ func (a adminAPIHandlers) SetUserPolicy(w http.ResponseWriter, r *http.Request)
|
||||
if err := globalIAMSys.SetUserPolicy(accessKey, policyName); err != nil {
|
||||
writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
|
||||
}
|
||||
|
||||
// Notify all other Minio peers to reload users
|
||||
for host, err := range globalNotificationSys.LoadUsers() {
|
||||
if err != nil {
|
||||
logger.GetReqInfo(ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// SetConfigHandler - PUT /minio/admin/v1/config
|
||||
|
||||
@@ -38,7 +38,7 @@ import (
|
||||
|
||||
var (
|
||||
configJSON = []byte(`{
|
||||
"version": "32",
|
||||
"version": "33",
|
||||
"credential": {
|
||||
"accessKey": "minio",
|
||||
"secretKey": "minio123"
|
||||
@@ -152,7 +152,6 @@ var (
|
||||
"streaming": {
|
||||
"enable": false,
|
||||
"clusterID": "",
|
||||
"clientID": "",
|
||||
"async": false,
|
||||
"maxPubAcksInflight": 0
|
||||
}
|
||||
@@ -278,7 +277,7 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
|
||||
|
||||
// Setup admin mgmt REST API handlers.
|
||||
adminRouter := mux.NewRouter()
|
||||
registerAdminRouter(adminRouter)
|
||||
registerAdminRouter(adminRouter, true)
|
||||
|
||||
return &adminXLTestBed{
|
||||
xlDirs: xlDirs,
|
||||
|
||||
@@ -641,7 +641,12 @@ func (h *healSequence) healDiskFormat() error {
|
||||
// Healing succeeded notify the peers to reload format and re-initialize disks.
|
||||
// We will not notify peers only if healing succeeded.
|
||||
if err == nil {
|
||||
peersReInitFormat(globalAdminPeers, h.settings.DryRun)
|
||||
for host, rerr := range globalNotificationSys.ReloadFormat(h.settings.DryRun) {
|
||||
if rerr != nil {
|
||||
logger.GetReqInfo(h.ctx).SetTags("peerAddress", host.String())
|
||||
logger.LogIf(h.ctx, rerr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Push format heal result
|
||||
|
||||
+32
-30
@@ -31,7 +31,7 @@ type adminAPIHandlers struct {
|
||||
}
|
||||
|
||||
// registerAdminRouter - Add handler functions for each service REST API routes.
|
||||
func registerAdminRouter(router *mux.Router) {
|
||||
func registerAdminRouter(router *mux.Router, enableIAM bool) {
|
||||
|
||||
adminAPI := adminAPIHandlers{}
|
||||
// Admin router
|
||||
@@ -69,42 +69,44 @@ func registerAdminRouter(router *mux.Router) {
|
||||
|
||||
/// Config operations
|
||||
|
||||
// Update credentials
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
|
||||
// Get config
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
|
||||
// Set config
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
|
||||
if enableIAM {
|
||||
// Update credentials
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(httpTraceHdrs(adminAPI.UpdateAdminCredentialsHandler))
|
||||
// Get config
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
|
||||
// Set config
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
|
||||
|
||||
// Get config keys/values
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
|
||||
// Set config keys/values
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
|
||||
// Get config keys/values
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKeysHandler))
|
||||
// Set config keys/values
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config-keys").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKeysHandler))
|
||||
|
||||
// -- IAM APIs --
|
||||
// -- IAM APIs --
|
||||
|
||||
// Add policy IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
|
||||
// Add policy IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
|
||||
|
||||
// Add user IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
|
||||
// Add user IAM
|
||||
adminV1Router.Methods(http.MethodPut).Path("/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-policy").HandlerFunc(httpTraceHdrs(adminAPI.SetUserPolicy)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("name", "{name:.*}")
|
||||
adminV1Router.Methods(http.MethodPut).Path("/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).
|
||||
Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
|
||||
|
||||
// Remove policy IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
|
||||
// Remove policy IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
|
||||
|
||||
// Remove user IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
// Remove user IAM
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
|
||||
|
||||
// List users
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
|
||||
// List users
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
|
||||
|
||||
// List policies
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
|
||||
// List policies
|
||||
adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
|
||||
}
|
||||
|
||||
// If none of the routes match.
|
||||
adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandler))
|
||||
// If none of the routes match, return error.
|
||||
adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
|
||||
}
|
||||
|
||||
+5
-39
@@ -45,29 +45,12 @@ func (rpcClient *AdminRPCClient) SignalService(signal serviceSignal) (err error)
|
||||
return rpcClient.Call(adminServiceName+".SignalService", &args, &reply)
|
||||
}
|
||||
|
||||
// ReInitFormat - re-initialize disk format, remotely.
|
||||
func (rpcClient *AdminRPCClient) ReInitFormat(dryRun bool) error {
|
||||
args := ReInitFormatArgs{DryRun: dryRun}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(adminServiceName+".ReInitFormat", &args, &reply)
|
||||
}
|
||||
|
||||
// ServerInfo - returns the server info of the server to which the RPC call is made.
|
||||
func (rpcClient *AdminRPCClient) ServerInfo() (sid ServerInfoData, err error) {
|
||||
err = rpcClient.Call(adminServiceName+".ServerInfo", &AuthArgs{}, &sid)
|
||||
return sid, err
|
||||
}
|
||||
|
||||
// GetConfig - returns config.json of the remote server.
|
||||
func (rpcClient *AdminRPCClient) GetConfig() ([]byte, error) {
|
||||
args := AuthArgs{}
|
||||
var reply []byte
|
||||
|
||||
err := rpcClient.Call(adminServiceName+".GetConfig", &args, &reply)
|
||||
return reply, err
|
||||
}
|
||||
|
||||
// StartProfiling - starts profiling in the remote server.
|
||||
func (rpcClient *AdminRPCClient) StartProfiling(profiler string) error {
|
||||
args := StartProfilingArgs{Profiler: profiler}
|
||||
@@ -125,9 +108,7 @@ func NewAdminRPCClient(host *xnet.Host) (*AdminRPCClient, error) {
|
||||
// commands like service stop and service restart.
|
||||
type adminCmdRunner interface {
|
||||
SignalService(s serviceSignal) error
|
||||
ReInitFormat(dryRun bool) error
|
||||
ServerInfo() (ServerInfoData, error)
|
||||
GetConfig() ([]byte, error)
|
||||
StartProfiling(string) error
|
||||
DownloadProfilingData() ([]byte, error)
|
||||
}
|
||||
@@ -149,6 +130,11 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
// Use first IPv4 instead of loopback address.
|
||||
localAddr = net.JoinHostPort(sortIPs(localIP4.ToSlice())[0], globalMinioPort)
|
||||
}
|
||||
if strings.HasPrefix(localAddr, "[::1]:") {
|
||||
// Use first IPv4 instead of loopback address.
|
||||
localAddr = net.JoinHostPort(localIP6.ToSlice()[0], globalMinioPort)
|
||||
}
|
||||
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
addr: localAddr,
|
||||
cmdRunner: localAdminClient{},
|
||||
@@ -169,26 +155,6 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
return adminPeerList
|
||||
}
|
||||
|
||||
// peersReInitFormat - reinitialize remote object layers to new format.
|
||||
func peersReInitFormat(peers adminPeers, dryRun bool) error {
|
||||
errs := make([]error, len(peers))
|
||||
|
||||
// Send ReInitFormat RPC call to all nodes.
|
||||
// for local adminPeer this is a no-op.
|
||||
wg := sync.WaitGroup{}
|
||||
for i, peer := range peers {
|
||||
wg.Add(1)
|
||||
go func(idx int, peer adminPeer) {
|
||||
defer wg.Done()
|
||||
if !peer.isLocal {
|
||||
errs[idx] = peer.cmdRunner.ReInitFormat(dryRun)
|
||||
}
|
||||
}(i, peer)
|
||||
}
|
||||
wg.Wait()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Initialize global adminPeer collection.
|
||||
func initGlobalAdminPeers(endpoints EndpointList) {
|
||||
globalAdminPeers = makeAdminPeers(endpoints)
|
||||
|
||||
@@ -68,23 +68,6 @@ func (receiver *adminRPCReceiver) DownloadProfilingData(args *AuthArgs, reply *[
|
||||
return
|
||||
}
|
||||
|
||||
// GetConfig - returns the config.json of this server.
|
||||
func (receiver *adminRPCReceiver) GetConfig(args *AuthArgs, reply *[]byte) (err error) {
|
||||
*reply, err = receiver.local.GetConfig()
|
||||
return err
|
||||
}
|
||||
|
||||
// ReInitFormatArgs - provides dry-run information to re-initialize format.json
|
||||
type ReInitFormatArgs struct {
|
||||
AuthArgs
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// ReInitFormat - re-init 'format.json'
|
||||
func (receiver *adminRPCReceiver) ReInitFormat(args *ReInitFormatArgs, reply *VoidReply) error {
|
||||
return receiver.local.ReInitFormat(args.DryRun)
|
||||
}
|
||||
|
||||
// NewAdminRPCServer - returns new admin RPC server.
|
||||
func NewAdminRPCServer() (*xrpc.Server, error) {
|
||||
rpcServer := xrpc.NewServer()
|
||||
|
||||
+1
-79
@@ -40,11 +40,8 @@ import (
|
||||
///////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalServiceSignalCh := globalServiceSignalCh
|
||||
defer func(sigChan chan serviceSignal) { globalServiceSignalCh = sigChan }(globalServiceSignalCh)
|
||||
globalServiceSignalCh = make(chan serviceSignal, 10)
|
||||
defer func() {
|
||||
globalServiceSignalCh = tmpGlobalServiceSignalCh
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
signal serviceSignal
|
||||
@@ -66,34 +63,6 @@ func testAdminCmdRunnerSignalService(t *testing.T, client adminCmdRunner) {
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerReInitFormat(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalObjectAPI := globalObjectAPI
|
||||
defer func() {
|
||||
globalObjectAPI = tmpGlobalObjectAPI
|
||||
}()
|
||||
|
||||
testCases := []struct {
|
||||
objectAPI ObjectLayer
|
||||
dryRun bool
|
||||
expectErr bool
|
||||
}{
|
||||
{&DummyObjectLayer{}, true, false},
|
||||
{&DummyObjectLayer{}, false, false},
|
||||
{nil, true, true},
|
||||
{nil, false, true},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
globalObjectAPI = testCase.objectAPI
|
||||
err := client.ReInitFormat(testCase.dryRun)
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalBootTime := globalBootTime
|
||||
tmpGlobalObjectAPI := globalObjectAPI
|
||||
@@ -140,33 +109,6 @@ func testAdminCmdRunnerServerInfo(t *testing.T, client adminCmdRunner) {
|
||||
}
|
||||
}
|
||||
|
||||
func testAdminCmdRunnerGetConfig(t *testing.T, client adminCmdRunner) {
|
||||
tmpGlobalServerConfig := globalServerConfig
|
||||
defer func() {
|
||||
globalServerConfig = tmpGlobalServerConfig
|
||||
}()
|
||||
|
||||
config := newServerConfig()
|
||||
|
||||
testCases := []struct {
|
||||
config *serverConfig
|
||||
expectErr bool
|
||||
}{
|
||||
{globalServerConfig, false},
|
||||
{config, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
globalServerConfig = testCase.config
|
||||
_, err := client.GetConfig()
|
||||
expectErr := (err != nil)
|
||||
|
||||
if expectErr != testCase.expectErr {
|
||||
t.Fatalf("case %v: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func newAdminRPCHTTPServerClient(t *testing.T) (*httptest.Server, *AdminRPCClient, *serverConfig) {
|
||||
rpcServer, err := NewAdminRPCServer()
|
||||
if err != nil {
|
||||
@@ -208,16 +150,6 @@ func TestAdminRPCClientSignalService(t *testing.T) {
|
||||
testAdminCmdRunnerSignalService(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientReInitFormat(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
defer func() {
|
||||
globalServerConfig = prevGlobalServerConfig
|
||||
}()
|
||||
|
||||
testAdminCmdRunnerReInitFormat(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientServerInfo(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
@@ -227,13 +159,3 @@ func TestAdminRPCClientServerInfo(t *testing.T) {
|
||||
|
||||
testAdminCmdRunnerServerInfo(t, rpcClient)
|
||||
}
|
||||
|
||||
func TestAdminRPCClientGetConfig(t *testing.T) {
|
||||
httpServer, rpcClient, prevGlobalServerConfig := newAdminRPCHTTPServerClient(t)
|
||||
defer httpServer.Close()
|
||||
defer func() {
|
||||
globalServerConfig = prevGlobalServerConfig
|
||||
}()
|
||||
|
||||
testAdminCmdRunnerGetConfig(t, rpcClient)
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ import (
|
||||
const (
|
||||
// Response request id.
|
||||
responseRequestIDKey = "x-amz-request-id"
|
||||
// Deployment id.
|
||||
responseDeploymentIDKey = "x-minio-deployment-id"
|
||||
)
|
||||
|
||||
// ObjectIdentifier carries key name for the object to delete.
|
||||
|
||||
@@ -87,6 +87,7 @@ const (
|
||||
ErrNoSuchBucketPolicy
|
||||
ErrNoSuchKey
|
||||
ErrNoSuchUpload
|
||||
ErrNoSuchVersion
|
||||
ErrNotImplemented
|
||||
ErrPreconditionFailed
|
||||
ErrRequestTimeTooSkewed
|
||||
@@ -447,6 +448,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNoSuchVersion: {
|
||||
Code: "NoSuchVersion",
|
||||
Description: "Indicates that the version ID specified in the request does not match an existing version.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrNotImplemented: {
|
||||
Code: "NotImplemented",
|
||||
Description: "A header you provided implies functionality that is not implemented",
|
||||
|
||||
+10
-1
@@ -568,13 +568,22 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
|
||||
}
|
||||
|
||||
// writeErrorRespone writes error headers
|
||||
func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
|
||||
func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL, browser bool) {
|
||||
switch errorCode {
|
||||
case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set("Retry-After", "120")
|
||||
case ErrAccessDenied:
|
||||
// The request is from browser and also if browser
|
||||
// is enabled we need to redirect.
|
||||
if browser && globalIsBrowserEnabled {
|
||||
w.Header().Set("Location", minioReservedBucketPath+reqURL.Path)
|
||||
w.WriteHeader(http.StatusTemporaryRedirect)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
apiError := getAPIError(errorCode)
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path, w.Header().Get(responseRequestIDKey))
|
||||
|
||||
+58
-41
@@ -19,10 +19,10 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
@@ -155,49 +155,66 @@ func getSessionToken(r *http.Request) (token string) {
|
||||
return r.URL.Query().Get("X-Amz-Security-Token")
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client and validate the token.
|
||||
func getClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
|
||||
stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
|
||||
if _, ok := jwtToken.Method.(*jwtgo.SigningMethodHMAC); !ok {
|
||||
return nil, fmt.Errorf("Unexpected signing method: %v", jwtToken.Header["alg"])
|
||||
}
|
||||
if err := jwtToken.Claims.Valid(); err != nil {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
if claims, ok := jwtToken.Claims.(jwtgo.MapClaims); ok {
|
||||
if _, ok = claims["accessKey"].(string); !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
// JWT token for x-amz-security-token is signed with admin
|
||||
// secret key, temporary credentials become invalid if
|
||||
// server admin credentials change. This is done to ensure
|
||||
// that clients cannot decode the token using the temp
|
||||
// secret keys and generate an entirely new claim by essentially
|
||||
// hijacking the policies. We need to make sure that this is
|
||||
// based an admin credential such that token cannot be decoded
|
||||
// on the client side and is treated like an opaque value.
|
||||
return []byte(globalServerConfig.GetCredential().SecretKey), nil
|
||||
}
|
||||
return nil, errAuthentication
|
||||
}
|
||||
// Fetch claims in the security token returned by the client, doesn't return
|
||||
// errors - upon errors the returned claims map will be empty.
|
||||
func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
|
||||
claims, _ := getClaimsFromToken(r)
|
||||
return claims
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client.
|
||||
func getClaimsFromToken(r *http.Request) (map[string]interface{}, error) {
|
||||
claims := make(map[string]interface{})
|
||||
token := getSessionToken(r)
|
||||
if token == "" {
|
||||
return nil, ErrNone
|
||||
return claims, nil
|
||||
}
|
||||
stsTokenCallback := func(jwtToken *jwtgo.Token) (interface{}, error) {
|
||||
// JWT token for x-amz-security-token is signed with admin
|
||||
// secret key, temporary credentials become invalid if
|
||||
// server admin credentials change. This is done to ensure
|
||||
// that clients cannot decode the token using the temp
|
||||
// secret keys and generate an entirely new claim by essentially
|
||||
// hijacking the policies. We need to make sure that this is
|
||||
// based an admin credential such that token cannot be decoded
|
||||
// on the client side and is treated like an opaque value.
|
||||
return []byte(globalServerConfig.GetCredential().SecretKey), nil
|
||||
}
|
||||
p := &jwtgo.Parser{
|
||||
ValidMethods: []string{
|
||||
jwtgo.SigningMethodHS256.Alg(),
|
||||
jwtgo.SigningMethodHS512.Alg(),
|
||||
},
|
||||
}
|
||||
jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !jtoken.Valid {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
v, ok := claims["accessKey"]
|
||||
if !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
if _, ok = v.(string); !ok {
|
||||
return nil, errInvalidAccessKeyID
|
||||
}
|
||||
return claims, nil
|
||||
}
|
||||
|
||||
// Fetch claims in the security token returned by the client and validate the token.
|
||||
func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
|
||||
token := getSessionToken(r)
|
||||
if token != "" && cred.AccessKey == "" {
|
||||
return nil, ErrNoAccessKey
|
||||
}
|
||||
if token != cred.SessionToken {
|
||||
if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
|
||||
return nil, ErrInvalidToken
|
||||
}
|
||||
p := &jwtgo.Parser{}
|
||||
jtoken, err := p.ParseWithClaims(token, jwtgo.MapClaims(claims), stsTokenCallback)
|
||||
claims, err := getClaimsFromToken(r)
|
||||
if err != nil {
|
||||
return nil, toAPIErrorCode(context.Background(), errAuthentication)
|
||||
}
|
||||
if !jtoken.Valid {
|
||||
return nil, toAPIErrorCode(context.Background(), errAuthentication)
|
||||
return nil, toAPIErrorCode(context.Background(), err)
|
||||
}
|
||||
return claims, ErrNone
|
||||
}
|
||||
@@ -233,6 +250,11 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
||||
return s3Err
|
||||
}
|
||||
|
||||
claims, s3Err := checkClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
// LocationConstraint is valid only for CreateBucketAction.
|
||||
var locationConstraint string
|
||||
if action == policy.CreateBucketAction {
|
||||
@@ -256,11 +278,6 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
|
||||
}
|
||||
|
||||
claims, s3Err := getClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
if cred.AccessKey == "" {
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
@@ -399,7 +416,7 @@ func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
a.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL)
|
||||
writeErrorResponse(w, ErrSignatureVersionNotSupported, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
|
||||
// isPutAllowed - check if PUT operation is allowed on the resource, this
|
||||
@@ -421,7 +438,7 @@ func isPutAllowed(atype authType, bucketName, objectName string, r *http.Request
|
||||
return s3Err
|
||||
}
|
||||
|
||||
claims, s3Err := getClaimsFromToken(r, cred)
|
||||
claims, s3Err := checkClaimsFromToken(r, cred)
|
||||
if s3Err != ErrNone {
|
||||
return s3Err
|
||||
}
|
||||
|
||||
@@ -59,19 +59,19 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
|
||||
func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectsV2")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "ListObjectsV2", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -81,14 +81,14 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
prefix, token, startAfter, delimiter, fetchOwner, maxKeys, _, errCode := getListObjectsV2Args(urlValues)
|
||||
|
||||
if errCode != ErrNone {
|
||||
writeErrorResponse(w, errCode, r.URL)
|
||||
writeErrorResponse(w, errCode, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the query params before beginning to serve the request.
|
||||
// fetch-owner is not validated since it is a boolean
|
||||
if s3Error := validateListObjectsArgs(prefix, token, delimiter, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listObjectsV2 := objectAPI.ListObjectsV2
|
||||
@@ -100,7 +100,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// Read the decompressed size from the meta.json.
|
||||
actualSize = listObjectsV2Info.Objects[i].GetActualSize()
|
||||
if actualSize < 0 {
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Set the info.Size to the actualSize.
|
||||
@@ -119,7 +119,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
|
||||
listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].DecryptedSize()
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -141,37 +141,37 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListObjectsV1")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "ListObjectsV1", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Extract all the litsObjectsV1 query params to their native values.
|
||||
prefix, marker, delimiter, maxKeys, _, s3Error := getListObjectsV1Args(r.URL.Query())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the maxKeys lowerbound. When maxKeys > 1000, S3 returns 1000 but
|
||||
// does not throw an error.
|
||||
if maxKeys < 0 {
|
||||
writeErrorResponse(w, ErrInvalidMaxKeys, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidMaxKeys, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
} // Validate all the query params before beginning to serve the request.
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listObjects := objectAPI.ListObjects
|
||||
@@ -183,7 +183,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
// marshaled into S3 compatible XML header.
|
||||
listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -193,7 +193,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
// Read the decompressed size from the meta.json.
|
||||
actualSize = listObjectsInfo.Objects[i].GetActualSize()
|
||||
if actualSize < 0 {
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidDecompressedSize, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Set the info.Size to the actualSize.
|
||||
@@ -202,7 +202,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
|
||||
listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].DecryptedSize()
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
+93
-79
@@ -89,19 +89,19 @@ func initFederatorBackend(objLayer ObjectLayer) {
|
||||
func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketLocation")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "GetBucketLocation", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -110,7 +110,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
getBucketInfo = api.CacheAPI().GetBucketInfo
|
||||
}
|
||||
if _, err := getBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -139,42 +139,42 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListMultipartUploads")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "ListMultipartUploads", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, _, s3Error := getBucketMultipartResources(r.URL.Query())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if maxUploads < 0 {
|
||||
writeErrorResponse(w, ErrInvalidMaxUploads, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidMaxUploads, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if keyMarker != "" {
|
||||
// Marker not common with prefix is not implemented.
|
||||
if !hasPrefix(keyMarker, prefix) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// generate response
|
||||
@@ -192,11 +192,11 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListBuckets")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "ListBuckets", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
listBuckets := objectAPI.ListBuckets
|
||||
@@ -206,7 +206,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListAllMyBucketsAction, "", ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// If etcd, dns federation configured list buckets from etcd.
|
||||
@@ -214,7 +214,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
if globalDNSConfig != nil {
|
||||
dnsBuckets, err := globalDNSConfig.List()
|
||||
if err != nil && err != dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
bucketSet := set.NewStringSet()
|
||||
@@ -233,7 +233,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
var err error
|
||||
bucketsInfo, err = listBuckets(ctx)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -250,14 +250,14 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteMultipleObjects")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "DeleteMultipleObjects", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -266,7 +266,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// In the event access is denied, a 200 response should still be returned
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if s3Error != ErrAccessDenied {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -274,14 +274,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Content-Length is required and should be non-zero
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Content-Md5 is requied should be set
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if _, ok := r.Header["Content-Md5"]; !ok {
|
||||
writeErrorResponse(w, ErrMissingContentMD5, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentMD5, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -297,7 +297,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Read incoming body XML bytes.
|
||||
if _, err := io.ReadFull(r.Body, deleteXMLBytes); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -305,7 +305,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
deleteObjects := &DeleteObjectsRequest{}
|
||||
if err := xml.Unmarshal(deleteXMLBytes, deleteObjects); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedXML, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedXML, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -313,7 +313,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
if globalWORMEnabled {
|
||||
// Not required to check whether given objects exist or not, because
|
||||
// DeleteMultipleObject is always successful irrespective of object existence.
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -397,11 +397,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucket")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "PutBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -409,21 +409,21 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.CreateBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Parse incoming location constraint.
|
||||
location, s3Error := parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if location sent by the client is valid, reject
|
||||
// requests which do not follow valid region requirements.
|
||||
if !isValidLocation(location) {
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -432,12 +432,12 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
// Proceed to creating a bucket.
|
||||
if err = objectAPI.MakeBucketWithLocation(ctx, bucket, location); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if err = globalDNSConfig.Put(bucket); err != nil {
|
||||
objectAPI.DeleteBucket(ctx, bucket)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -447,18 +447,18 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
|
||||
}
|
||||
writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL)
|
||||
writeErrorResponse(w, ErrBucketAlreadyOwnedByYou, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucketWithLocation(ctx, bucket, location)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -475,30 +475,37 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PostPolicyBucket")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "PostPolicyBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if !objectAPI.IsEncryptionSupported() && hasServerSideEncryptionHeader(r.Header) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
bucket := mux.Vars(r)["bucket"]
|
||||
|
||||
// Require Content-Length to be set in the request
|
||||
size := r.ContentLength
|
||||
if size < 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
resource, err := getResource(r.URL.Path, r.Host, globalDomainName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Make sure that the URL does not contain object name.
|
||||
if bucket != filepath.Clean(resource[1:]) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -507,7 +514,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
reader, err := r.MultipartReader()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -515,7 +522,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
form, err := reader.ReadForm(maxFormMemory)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -526,13 +533,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if file is provided, error out otherwise.
|
||||
if fileBody == nil {
|
||||
writeErrorResponse(w, ErrPOSTFileRequired, r.URL)
|
||||
writeErrorResponse(w, ErrPOSTFileRequired, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -554,7 +561,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if successRedirect != "" {
|
||||
redirectURL, err = url.Parse(successRedirect)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -562,60 +569,67 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// Verify policy signature.
|
||||
apiErr := doesPolicySignatureMatch(formValues)
|
||||
if apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure formValues adhere to policy restrictions.
|
||||
if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure that the object size is within expected range, also the file size
|
||||
// should not exceed the maximum single Put size (5 GiB)
|
||||
lengthRange := postPolicyForm.Conditions.ContentLengthRange
|
||||
if lengthRange.Valid {
|
||||
if fileSize < lengthRange.Min {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL)
|
||||
// Handle policy if it is set.
|
||||
if len(policyBytes) > 0 {
|
||||
postPolicyForm, err := parsePostPolicyForm(string(policyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPOSTRequest, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL)
|
||||
// Make sure formValues adhere to policy restrictions.
|
||||
if apiErr = checkPostPolicy(formValues, postPolicyForm); apiErr != ErrNone {
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure that the object size is within expected range, also the file size
|
||||
// should not exceed the maximum single Put size (5 GiB)
|
||||
lengthRange := postPolicyForm.Conditions.ContentLengthRange
|
||||
if lengthRange.Valid {
|
||||
if fileSize < lengthRange.Min {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooSmall), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, errDataTooLarge), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract metadata to be saved from received Form.
|
||||
metadata := make(map[string]string)
|
||||
err = extractMetadataFromMap(ctx, formValues, metadata)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
rawReader := hashReader
|
||||
pReader := NewPutObjReader(rawReader, nil, nil)
|
||||
var objectEncryptionKey []byte
|
||||
|
||||
if globalAutoEncryption && !crypto.SSEC.IsRequested(r.Header) {
|
||||
r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
}
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if hasServerSideEncryptionHeader(formValues) && !hasSuffix(object, slashSeparator) { // handle SSE-C and SSE-S3 requests
|
||||
var reader io.Reader
|
||||
@@ -623,19 +637,19 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if crypto.SSEC.IsRequested(formValues) {
|
||||
key, err = ParseSSECustomerHeader(formValues)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
reader, objectEncryptionKey, err = newEncryptReader(hashReader, key, bucket, object, metadata, crypto.S3.IsRequested(formValues))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
info := ObjectInfo{Size: fileSize}
|
||||
hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize) // do not try to verify encrypted content
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
pReader = NewPutObjReader(rawReader, hashReader, objectEncryptionKey)
|
||||
@@ -644,7 +658,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, metadata, ObjectOptions{})
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -703,7 +717,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "HeadBucket")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "HeadBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
@@ -735,19 +749,19 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucket")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "DeleteBucket", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -757,7 +771,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
// Attempt to delete bucket.
|
||||
if err := deleteBucket(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -769,7 +783,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
if err := globalDNSConfig.Delete(bucket); err != nil {
|
||||
// Deleting DNS entry failed, attempt to create the bucket again.
|
||||
objectAPI.MakeBucketWithLocation(ctx, bucket, "")
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,30 +44,30 @@ var errNoSuchNotifications = errors.New("The specified bucket does not have buck
|
||||
func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketNotification")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "GetBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objAPI.GetBucketInfo(ctx, bucketName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -76,7 +76,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
if err != nil {
|
||||
// Ignore errNoSuchNotifications to comply with AWS S3.
|
||||
if err != errNoSuchNotifications {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
|
||||
notificationBytes, err := xml.Marshal(nConfig)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -102,16 +102,16 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketNotification")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "PutBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objectAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -119,19 +119,19 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
_, err := objectAPI.GetBucketInfo(ctx, bucketName)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucketNotification always needs a Content-Length.
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -143,12 +143,12 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
apiErr = toAPIErrorCode(ctx, err)
|
||||
}
|
||||
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = saveNotificationConfig(ctx, objectAPI, bucketName, config); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -164,24 +164,28 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "ListenBucketNotification")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "ListenBucketNotification", mustGetClaimsFromToken(r))
|
||||
|
||||
// Validate if bucket exists.
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
if !objAPI.IsNotificationSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if !objAPI.IsListenBucketSupported() {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
vars := mux.Vars(r)
|
||||
bucketName := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -189,11 +193,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
var prefix string
|
||||
if len(values["prefix"]) > 1 {
|
||||
writeErrorResponse(w, ErrFilterNamePrefix, r.URL)
|
||||
writeErrorResponse(w, ErrFilterNamePrefix, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
if len(values["prefix"]) == 1 {
|
||||
if err := event.ValidateFilterRuleValue(values["prefix"][0]); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -202,11 +206,11 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
var suffix string
|
||||
if len(values["suffix"]) > 1 {
|
||||
writeErrorResponse(w, ErrFilterNameSuffix, r.URL)
|
||||
writeErrorResponse(w, ErrFilterNameSuffix, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
if len(values["suffix"]) == 1 {
|
||||
if err := event.ValidateFilterRuleValue(values["suffix"][0]); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -219,7 +223,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
for _, s := range values["events"] {
|
||||
eventName, err := event.ParseName(s)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -227,19 +231,19 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
}
|
||||
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucketName); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
host, err := xnet.ParseHost(r.RemoteAddr)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
target, err := target.NewHTTPClientTarget(*host, w)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -247,7 +251,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
if err = globalNotificationSys.AddRemoteTarget(bucketName, target, rulesMap); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
defer globalNotificationSys.RemoveRemoteTarget(bucketName, target.ID())
|
||||
@@ -255,13 +259,13 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = SaveListener(objAPI, bucketName, eventNames, pattern, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -271,7 +275,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
if err = RemoveListener(objAPI, bucketName, target.ID(), *thisAddr); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("target", target.ID().Name)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -40,11 +40,11 @@ const (
|
||||
func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "PutBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "PutBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -52,43 +52,43 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is missing.
|
||||
// PutBucketPolicy always needs Content-Length.
|
||||
if r.ContentLength <= 0 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
if r.ContentLength > maxBucketPolicySize {
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL)
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Version in policy must not be empty
|
||||
if bucketPolicy.Version == "" {
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL)
|
||||
writeErrorResponse(w, ErrMalformedPolicy, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err = objAPI.SetBucketPolicy(ctx, bucket, bucketPolicy); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -103,11 +103,11 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "DeleteBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "DeleteBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -115,18 +115,18 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
if err := objAPI.DeleteBucketPolicy(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -141,11 +141,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "GetBucketPolicy")
|
||||
|
||||
defer logger.AuditLog(ctx, w, r)
|
||||
defer logger.AuditLog(w, r, "GetBucketPolicy", mustGetClaimsFromToken(r))
|
||||
|
||||
objAPI := api.ObjectAPI()
|
||||
if objAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -153,26 +153,26 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
bucket := vars["bucket"]
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
writeErrorResponse(w, s3Error, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Check if bucket exists.
|
||||
if _, err := objAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
// Read bucket access policy.
|
||||
bucketPolicy, err := objAPI.GetBucketPolicy(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
policyData, err := json.Marshal(bucketPolicy)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
+5
-1
@@ -71,9 +71,13 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
|
||||
// Get all CA file names.
|
||||
var caFiles []string
|
||||
fis, err := readDir(certsCAsDir)
|
||||
if err != nil {
|
||||
if err != nil && err != errFileNotFound {
|
||||
return nil, err
|
||||
}
|
||||
// Return success if CA's directory is missing.
|
||||
if err == errFileNotFound {
|
||||
return nil, nil
|
||||
}
|
||||
for _, fi := range fis {
|
||||
// Skip all directories.
|
||||
if hasSuffix(fi, slashSeparator) {
|
||||
|
||||
+2
-1
@@ -223,7 +223,8 @@ func TestGetRootCAs(t *testing.T) {
|
||||
certCAsDir string
|
||||
expectedErr error
|
||||
}{
|
||||
{"nonexistent-dir", errFileNotFound},
|
||||
// ignores non-existent directories.
|
||||
{"nonexistent-dir", nil},
|
||||
// Ignores directories.
|
||||
{dir1, nil},
|
||||
// Ignore empty directory.
|
||||
|
||||
+37
-21
@@ -30,8 +30,9 @@ import (
|
||||
dns2 "github.com/miekg/dns"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/logger/target/console"
|
||||
"github.com/minio/minio/cmd/logger/target/http"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/dns"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
@@ -54,31 +55,56 @@ func loadLoggers() {
|
||||
auditEndpoint, ok := os.LookupEnv("MINIO_AUDIT_LOGGER_HTTP_ENDPOINT")
|
||||
if ok {
|
||||
// Enable audit HTTP logging through ENV.
|
||||
logger.AddAuditTarget(logger.NewHTTP(auditEndpoint, NewCustomHTTPTransport()))
|
||||
logger.AddAuditTarget(http.New(auditEndpoint, NewCustomHTTPTransport()))
|
||||
}
|
||||
|
||||
loggerEndpoint, ok := os.LookupEnv("MINIO_LOGGER_HTTP_ENDPOINT")
|
||||
if ok {
|
||||
// Enable HTTP logging through ENV.
|
||||
logger.AddTarget(logger.NewHTTP(loggerEndpoint, NewCustomHTTPTransport()))
|
||||
logger.AddTarget(http.New(loggerEndpoint, NewCustomHTTPTransport()))
|
||||
} else {
|
||||
for _, l := range globalServerConfig.Logger.HTTP {
|
||||
if l.Enabled {
|
||||
// Enable http logging
|
||||
logger.AddTarget(logger.NewHTTP(l.Endpoint, NewCustomHTTPTransport()))
|
||||
logger.AddTarget(http.New(l.Endpoint, NewCustomHTTPTransport()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if globalServerConfig.Logger.Console.Enabled {
|
||||
// Enable console logging
|
||||
logger.AddTarget(logger.NewConsole())
|
||||
logger.AddTarget(console.New())
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func handleCommonCmdArgs(ctx *cli.Context) {
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
globalCLIContext.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if globalCLIContext.JSON {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
globalCLIContext.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if globalCLIContext.Quiet {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Get anonymous flag from command line argument.
|
||||
globalCLIContext.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous")
|
||||
if globalCLIContext.Anonymous {
|
||||
logger.EnableAnonymous()
|
||||
}
|
||||
|
||||
// Fetch address option
|
||||
globalCLIContext.Addr = ctx.GlobalString("address")
|
||||
if globalCLIContext.Addr == "" || globalCLIContext.Addr == ":"+globalMinioDefaultPort {
|
||||
globalCLIContext.Addr = ctx.String("address")
|
||||
}
|
||||
|
||||
var configDir string
|
||||
|
||||
switch {
|
||||
@@ -219,14 +245,18 @@ func handleCommonEnvVars() {
|
||||
minioEndpointsEnv, ok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if ok {
|
||||
minioEndpoints := strings.Split(minioEndpointsEnv, ",")
|
||||
globalDomainIPs = set.NewStringSet()
|
||||
for i, ip := range minioEndpoints {
|
||||
if net.ParseIP(ip) == nil {
|
||||
logger.FatalIf(errInvalidArgument, "Unable to initialize Minio server with invalid MINIO_PUBLIC_IPS[%d]: %s", i, ip)
|
||||
}
|
||||
globalDomainIPs.Add(ip)
|
||||
}
|
||||
updateDomainIPs(set.CreateStringSet(minioEndpoints...))
|
||||
} else {
|
||||
// Add found interfaces IP address to global domain IPS,
|
||||
// loopback addresses will be naturally dropped.
|
||||
updateDomainIPs(localIP4)
|
||||
}
|
||||
|
||||
if globalDomainName != "" && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil {
|
||||
var err error
|
||||
globalDNSConfig, err = dns.NewCoreDNS(globalDomainName, globalDomainIPs, globalMinioPort, globalEtcdClient)
|
||||
@@ -316,20 +346,6 @@ func handleCommonEnvVars() {
|
||||
globalWORMEnabled = bool(wormFlag)
|
||||
}
|
||||
|
||||
kmsConf, err := crypto.NewVaultConfig()
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize hashicorp vault")
|
||||
}
|
||||
if kmsConf.Vault.Endpoint != "" {
|
||||
kms, err := crypto.NewVault(kmsConf)
|
||||
if err != nil {
|
||||
logger.Fatal(err, "Unable to initialize KMS")
|
||||
}
|
||||
globalKMS = kms
|
||||
globalKMSKeyID = kmsConf.Vault.Key.Name
|
||||
globalKMSConfig = kmsConf
|
||||
}
|
||||
|
||||
if compress := os.Getenv("MINIO_COMPRESS"); compress != "" {
|
||||
globalIsCompressionEnabled = strings.EqualFold(compress, "true")
|
||||
}
|
||||
|
||||
+10
-12
@@ -43,9 +43,9 @@ import (
|
||||
// 6. Make changes in config-current_test.go for any test change
|
||||
|
||||
// Config version
|
||||
const serverConfigVersion = "32"
|
||||
const serverConfigVersion = "33"
|
||||
|
||||
type serverConfig = serverConfigV32
|
||||
type serverConfig = serverConfigV33
|
||||
|
||||
var (
|
||||
// globalServerConfig server config.
|
||||
@@ -271,8 +271,8 @@ func (s *serverConfig) loadFromEnvs() {
|
||||
s.SetCacheConfig(globalCacheDrives, globalCacheExcludes, globalCacheExpiry, globalCacheMaxUse)
|
||||
}
|
||||
|
||||
if globalKMS != nil {
|
||||
s.KMS = globalKMSConfig
|
||||
if err := Environment.LookupKMSConfig(s.KMS); err != nil {
|
||||
logger.FatalIf(err, "Unable to setup the KMS")
|
||||
}
|
||||
|
||||
if globalIsEnvCompression {
|
||||
@@ -534,12 +534,8 @@ func (s *serverConfig) loadToCachedConfigs() {
|
||||
globalCacheExpiry = cacheConf.Expiry
|
||||
globalCacheMaxUse = cacheConf.MaxUse
|
||||
}
|
||||
if globalKMS == nil {
|
||||
globalKMSConfig = s.KMS
|
||||
if kms, err := crypto.NewVault(globalKMSConfig); err == nil {
|
||||
globalKMS = kms
|
||||
globalKMSKeyID = globalKMSConfig.Vault.Key.Name
|
||||
}
|
||||
if err := Environment.LookupKMSConfig(s.KMS); err != nil {
|
||||
logger.FatalIf(err, "Unable to setup the KMS")
|
||||
}
|
||||
|
||||
if !globalIsCompressionEnabled {
|
||||
@@ -556,8 +552,10 @@ func (s *serverConfig) loadToCachedConfigs() {
|
||||
if globalPolicyOPA == nil {
|
||||
if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
|
||||
globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
|
||||
URL: s.Policy.OPA.URL,
|
||||
AuthToken: s.Policy.OPA.AuthToken,
|
||||
URL: s.Policy.OPA.URL,
|
||||
AuthToken: s.Policy.OPA.AuthToken,
|
||||
Transport: NewCustomHTTPTransport(),
|
||||
CloseRespFn: CloseResponse,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
|
||||
|
||||
// Test 12 - Test NATS
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
|
||||
// Test 13 - Test ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
|
||||
|
||||
+9
-1
@@ -74,7 +74,15 @@ func (config *ConfigDir) GetCADir() string {
|
||||
|
||||
// Create - creates configuration directory tree.
|
||||
func (config *ConfigDir) Create() error {
|
||||
return os.MkdirAll(config.GetCADir(), 0700)
|
||||
err := os.MkdirAll(config.GetCADir(), 0700)
|
||||
// It is possible in kubernetes like deployments this directory
|
||||
// is already mounted and is not writable, ignore any write errors.
|
||||
if err != nil {
|
||||
if os.IsPermission(err) {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// GetMinioConfigFile - returns absolute path of config.json file.
|
||||
|
||||
+94
-2
@@ -2413,8 +2413,64 @@ func migrateV27ToV28() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Migrates '.minio.sys/config.json' to v32.
|
||||
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
|
||||
defer func() {
|
||||
// Rename config.json to config.json.deprecated only upon
|
||||
// success of this function.
|
||||
if err == nil {
|
||||
os.Rename(getConfigFile(), getConfigFile()+".deprecated")
|
||||
}
|
||||
}()
|
||||
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
// Construct path to config.json for the given bucket.
|
||||
transactionConfigFile := configFile + ".transaction"
|
||||
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
// Verify if backend already has the file.
|
||||
if err = checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
|
||||
return err
|
||||
} // if errConfigNotFound proceed to migrate..
|
||||
|
||||
var config = &serverConfig{}
|
||||
if _, err = Load(getConfigFile(), config); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Read from deprecate file as well if necessary.
|
||||
if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return saveServerConfig(context.Background(), objAPI, config)
|
||||
}
|
||||
|
||||
// Migrates '.minio.sys/config.json' to v33.
|
||||
func migrateMinioSysConfig(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
// Construct path to config.json for the given bucket.
|
||||
transactionConfigFile := configFile + ".transaction"
|
||||
|
||||
// As object layer's GetObject() and PutObject() take respective lock on minioMetaBucket
|
||||
// and configFile, take a transaction lock to avoid data race between readConfig()
|
||||
// and saveConfig().
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, transactionConfigFile)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
if err := migrateV27ToV28MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -2427,7 +2483,10 @@ func migrateMinioSysConfig(objAPI ObjectLayer) error {
|
||||
if err := migrateV30ToV31MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
return migrateV31ToV32MinioSys(objAPI)
|
||||
if err := migrateV31ToV32MinioSys(objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
return migrateV32ToV33MinioSys(objAPI)
|
||||
}
|
||||
|
||||
func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
|
||||
@@ -2623,3 +2682,36 @@ func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
|
||||
ok, data, err := checkConfigVersion(objAPI, configFile, "32")
|
||||
if err == errConfigNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config file. %v", err)
|
||||
}
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfg := &serverConfigV33{}
|
||||
if err = json.Unmarshal(data, cfg); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg.Version = "33"
|
||||
|
||||
data, err = json.Marshal(cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = saveConfig(context.Background(), objAPI, configFile, data); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from 32 to 33 . %v", err)
|
||||
}
|
||||
|
||||
logger.Info(configMigrateMSGTemplate, configFile, "32", "33")
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -159,8 +159,8 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test if a config migration from v2 to v30 is successfully done
|
||||
func TestServerConfigMigrateV2toV30(t *testing.T) {
|
||||
// Test if a config migration from v2 to v33 is successfully done
|
||||
func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
rootPath, err := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
@@ -892,3 +892,47 @@ type serverConfigV32 struct {
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS.
|
||||
type serverConfigV33 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Worm BoolFlag `json:"worm"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache CacheConfig `json:"cache"`
|
||||
|
||||
// KMS configuration
|
||||
KMS crypto.KMSConfig `json:"kms"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger loggerConfig `json:"logger"`
|
||||
|
||||
// Compression configuration
|
||||
Compression compressionConfig `json:"compress"`
|
||||
|
||||
// OpenID configuration
|
||||
OpenID struct {
|
||||
// JWKS validator config.
|
||||
JWKS validator.JWKSArgs `json:"jwks"`
|
||||
} `json:"openid"`
|
||||
|
||||
// External policy enforcements.
|
||||
Policy struct {
|
||||
// OPA configuration.
|
||||
OPA iampolicy.OpaArgs `json:"opa"`
|
||||
|
||||
// Add new external policy enforcements here.
|
||||
} `json:"policy"`
|
||||
}
|
||||
|
||||
@@ -156,30 +156,6 @@ func NewConfigSys() *ConfigSys {
|
||||
return &ConfigSys{}
|
||||
}
|
||||
|
||||
// Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
|
||||
func migrateConfigToMinioSys(objAPI ObjectLayer) error {
|
||||
defer os.Rename(getConfigFile(), getConfigFile()+".deprecated")
|
||||
|
||||
configFile := path.Join(minioConfigPrefix, minioConfigFile)
|
||||
// Verify if backend already has the file.
|
||||
if err := checkConfig(context.Background(), objAPI, configFile); err != errConfigNotFound {
|
||||
return err
|
||||
} // if errConfigNotFound proceed to migrate..
|
||||
|
||||
var config = &serverConfig{}
|
||||
if _, err := Load(getConfigFile(), config); err != nil {
|
||||
if !os.IsNotExist(err) {
|
||||
return err
|
||||
}
|
||||
// Read from deprecate file as well if necessary.
|
||||
if _, err = Load(getConfigFile()+".deprecated", config); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return saveServerConfig(context.Background(), objAPI, config)
|
||||
}
|
||||
|
||||
// Initialize and load config from remote etcd or local config directory
|
||||
func initConfig(objAPI ObjectLayer) error {
|
||||
if objAPI == nil {
|
||||
|
||||
@@ -22,16 +22,16 @@ import (
|
||||
)
|
||||
|
||||
// Writes S3 compatible copy part range error.
|
||||
func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL) {
|
||||
func writeCopyPartErr(w http.ResponseWriter, err error, url *url.URL, browser bool) {
|
||||
switch err {
|
||||
case errInvalidRange:
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRange, url)
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRange, url, browser)
|
||||
return
|
||||
case errInvalidRangeSource:
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url)
|
||||
writeErrorResponse(w, ErrInvalidCopyPartRangeSource, url, browser)
|
||||
return
|
||||
default:
|
||||
writeErrorResponse(w, ErrInternalError, url)
|
||||
writeErrorResponse(w, ErrInternalError, url, browser)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,5 +16,6 @@ package crypto
|
||||
|
||||
// KMSConfig has the KMS config for hashicorp vault
|
||||
type KMSConfig struct {
|
||||
Vault VaultConfig `json:"vault"`
|
||||
AutoEncryption bool `json:"-"`
|
||||
Vault VaultConfig `json:"vault"`
|
||||
}
|
||||
|
||||
+123
-179
@@ -19,195 +19,133 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
vault "github.com/hashicorp/vault/api"
|
||||
)
|
||||
|
||||
const (
|
||||
// vaultEndpointEnv Vault endpoint environment variable
|
||||
vaultEndpointEnv = "MINIO_SSE_VAULT_ENDPOINT"
|
||||
// vaultAuthTypeEnv type of vault auth to be used
|
||||
vaultAuthTypeEnv = "MINIO_SSE_VAULT_AUTH_TYPE"
|
||||
// vaultAppRoleIDEnv Vault AppRole ID environment variable
|
||||
vaultAppRoleIDEnv = "MINIO_SSE_VAULT_APPROLE_ID"
|
||||
// vaultAppSecretIDEnv Vault AppRole Secret environment variable
|
||||
vaultAppSecretIDEnv = "MINIO_SSE_VAULT_APPROLE_SECRET"
|
||||
// vaultKeyVersionEnv Vault Key Version environment variable
|
||||
vaultKeyVersionEnv = "MINIO_SSE_VAULT_KEY_VERSION"
|
||||
// vaultKeyNameEnv Vault Encryption Key Name environment variable
|
||||
vaultKeyNameEnv = "MINIO_SSE_VAULT_KEY_NAME"
|
||||
|
||||
// vaultCAPath is the path to a directory of PEM-encoded CA
|
||||
// cert files to verify the Vault server SSL certificate.
|
||||
vaultCAPath = "MINIO_SSE_VAULT_CAPATH"
|
||||
)
|
||||
|
||||
var (
|
||||
//ErrKMSAuthLogin is raised when there is a failure authenticating to KMS
|
||||
ErrKMSAuthLogin = errors.New("Vault service did not return auth info")
|
||||
)
|
||||
|
||||
// VaultKey represents vault encryption key-ring.
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"` // The name of the encryption key-ring
|
||||
Version int `json:"version"` // The key version
|
||||
}
|
||||
|
||||
// VaultAuth represents vault authentication type.
|
||||
// Currently the only supported authentication type is AppRole.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"` // The authentication type
|
||||
AppRole VaultAppRole `json:"approle"` // The AppRole authentication credentials
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault AppRole authentication credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"` // The AppRole access ID
|
||||
Secret string `json:"secret"` // The AppRole secret
|
||||
}
|
||||
|
||||
// VaultConfig represents vault configuration.
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"` // The vault API endpoint as URL
|
||||
CAPath string `json:"-"` // The path to PEM-encoded certificate files used for mTLS. Currently not used in config file.
|
||||
Auth VaultAuth `json:"auth"` // The vault authentication configuration
|
||||
Key VaultKey `json:"key-id"` // The named key used for key-generation / decryption.
|
||||
Namespace string `json:"-"` // The vault namespace of enterprise vault instances
|
||||
}
|
||||
|
||||
// vaultService represents a connection to a vault KMS.
|
||||
type vaultService struct {
|
||||
config *VaultConfig
|
||||
client *vault.Client
|
||||
leaseDuration time.Duration
|
||||
}
|
||||
|
||||
// return transit secret engine's path for generate data key operation
|
||||
func (v *vaultService) genDataKeyEndpoint(key string) string {
|
||||
return "/transit/datakey/plaintext/" + key
|
||||
var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
|
||||
|
||||
// empty/default vault configuration used to check whether a particular is empty.
|
||||
var emptyVaultConfig = VaultConfig{}
|
||||
|
||||
// IsEmpty returns true if the vault config struct is an
|
||||
// empty configuration.
|
||||
func (v *VaultConfig) IsEmpty() bool { return *v == emptyVaultConfig }
|
||||
|
||||
// Verify returns a nil error if the vault configuration
|
||||
// is valid. A valid configuration is either empty or
|
||||
// contains valid non-default values.
|
||||
func (v *VaultConfig) Verify() (err error) {
|
||||
if v.IsEmpty() {
|
||||
return // an empty configuration is valid
|
||||
}
|
||||
switch {
|
||||
case v.Endpoint == "":
|
||||
err = errors.New("crypto: missing hashicorp vault endpoint")
|
||||
case strings.ToLower(v.Auth.Type) != "approle":
|
||||
err = fmt.Errorf("crypto: invalid hashicorp vault authentication type: %s is not supported", v.Auth.Type)
|
||||
case v.Auth.AppRole.ID == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppRole ID")
|
||||
case v.Auth.AppRole.Secret == "":
|
||||
err = errors.New("crypto: missing hashicorp vault AppSecret ID")
|
||||
case v.Key.Name == "":
|
||||
err = errors.New("crypto: missing hashicorp vault key name")
|
||||
case v.Key.Version < 0:
|
||||
err = errors.New("crypto: invalid hashicorp vault key version: The key version must not be negative")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// return transit secret engine's path for decrypt operation
|
||||
func (v *vaultService) decryptEndpoint(key string) string {
|
||||
return "/transit/decrypt/" + key
|
||||
}
|
||||
// NewVault initializes Hashicorp Vault KMS by authenticating
|
||||
// to Vault with the credentials in config and gets a client
|
||||
// token for future api calls.
|
||||
func NewVault(config VaultConfig) (KMS, error) {
|
||||
if config.IsEmpty() {
|
||||
return nil, errors.New("crypto: the hashicorp vault configuration must not be empty")
|
||||
}
|
||||
if err := config.Verify(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// VaultKey represents vault encryption key-id name & version
|
||||
type VaultKey struct {
|
||||
Name string `json:"name"`
|
||||
Version int `json:"version"`
|
||||
}
|
||||
vaultCfg := vault.Config{Address: config.Endpoint}
|
||||
if err := vaultCfg.ConfigureTLS(&vault.TLSConfig{CAPath: config.CAPath}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
client, err := vault.NewClient(&vaultCfg)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.Namespace != "" {
|
||||
client.SetNamespace(config.Namespace)
|
||||
}
|
||||
|
||||
// VaultAuth represents vault auth type to use. For now, AppRole is the only supported
|
||||
// auth type.
|
||||
type VaultAuth struct {
|
||||
Type string `json:"type"`
|
||||
AppRole VaultAppRole `json:"approle"`
|
||||
}
|
||||
|
||||
// VaultAppRole represents vault approle credentials
|
||||
type VaultAppRole struct {
|
||||
ID string `json:"id"`
|
||||
Secret string `json:"secret"`
|
||||
}
|
||||
|
||||
// VaultConfig holds config required to start vault service
|
||||
type VaultConfig struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Auth VaultAuth `json:"auth"`
|
||||
Key VaultKey `json:"key-id"`
|
||||
}
|
||||
|
||||
// validate whether all required env variables needed to start vault service have
|
||||
// been set
|
||||
func validateVaultConfig(c *VaultConfig) error {
|
||||
if c.Endpoint == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault endpoint - %s is empty", vaultEndpointEnv)
|
||||
payload := map[string]interface{}{
|
||||
"role_id": config.Auth.AppRole.ID,
|
||||
"secret_id": config.Auth.AppRole.Secret,
|
||||
}
|
||||
if strings.ToLower(c.Auth.Type) != "approle" {
|
||||
return fmt.Errorf("Unsupported hashicorp vault auth type - %s", vaultAuthTypeEnv)
|
||||
}
|
||||
if c.Auth.AppRole.ID == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppRole ID - %s is empty", vaultAppRoleIDEnv)
|
||||
}
|
||||
if c.Auth.AppRole.Secret == "" {
|
||||
return fmt.Errorf("Missing hashicorp vault AppSecret ID - %s is empty", vaultAppSecretIDEnv)
|
||||
}
|
||||
if c.Key.Name == "" {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyNameEnv)
|
||||
}
|
||||
if c.Key.Version < 0 {
|
||||
return fmt.Errorf("Invalid value set in environment variable %s", vaultKeyVersionEnv)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// authenticate to vault with app role id and app role secret, and get a client access token, lease duration
|
||||
func getVaultAccessToken(client *vault.Client, appRoleID, appSecret string) (token string, duration int, err error) {
|
||||
data := map[string]interface{}{
|
||||
"role_id": appRoleID,
|
||||
"secret_id": appSecret,
|
||||
}
|
||||
resp, e := client.Logical().Write("auth/approle/login", data)
|
||||
if e != nil {
|
||||
return token, duration, e
|
||||
resp, err := client.Logical().Write("auth/approle/login", payload)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.Auth == nil {
|
||||
return token, duration, ErrKMSAuthLogin
|
||||
return nil, ErrKMSAuthLogin
|
||||
}
|
||||
return resp.Auth.ClientToken, resp.Auth.LeaseDuration, nil
|
||||
|
||||
client.SetToken(resp.Auth.ClientToken)
|
||||
v := &vaultService{client: client, config: &config, leaseDuration: time.Duration(resp.Auth.LeaseDuration)}
|
||||
v.renewToken()
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// NewVaultConfig sets KMSConfig from environment
|
||||
// variables and performs validations.
|
||||
func NewVaultConfig() (KMSConfig, error) {
|
||||
kc := KMSConfig{}
|
||||
endpoint := os.Getenv(vaultEndpointEnv)
|
||||
roleID := os.Getenv(vaultAppRoleIDEnv)
|
||||
roleSecret := os.Getenv(vaultAppSecretIDEnv)
|
||||
keyName := os.Getenv(vaultKeyNameEnv)
|
||||
keyVersion := 0
|
||||
authType := "approle"
|
||||
if versionStr := os.Getenv(vaultKeyVersionEnv); versionStr != "" {
|
||||
version, err := strconv.Atoi(versionStr)
|
||||
if err != nil {
|
||||
return kc, fmt.Errorf("Unable to parse %s value (`%s`)", vaultKeyVersionEnv, versionStr)
|
||||
}
|
||||
keyVersion = version
|
||||
}
|
||||
// return if none of the vault env variables are configured
|
||||
if (endpoint == "") && (roleID == "") && (roleSecret == "") && (keyName == "") && (keyVersion == 0) {
|
||||
return kc, nil
|
||||
}
|
||||
c := VaultConfig{
|
||||
Endpoint: endpoint,
|
||||
Auth: VaultAuth{
|
||||
Type: authType,
|
||||
AppRole: VaultAppRole{
|
||||
ID: roleID,
|
||||
Secret: roleSecret,
|
||||
},
|
||||
},
|
||||
Key: VaultKey{
|
||||
Version: keyVersion,
|
||||
Name: keyName,
|
||||
},
|
||||
}
|
||||
if err := validateVaultConfig(&c); err != nil {
|
||||
return kc, err
|
||||
}
|
||||
kc.Vault = c
|
||||
return kc, nil
|
||||
}
|
||||
|
||||
// NewVault initializes Hashicorp Vault KMS by
|
||||
// authenticating to Vault with the credentials in KMSConfig,
|
||||
// and gets a client token for future api calls.
|
||||
func NewVault(kmsConf KMSConfig) (KMS, error) {
|
||||
config := kmsConf.Vault
|
||||
vconfig := &vault.Config{
|
||||
Address: config.Endpoint,
|
||||
}
|
||||
if err := vconfig.ConfigureTLS(&vault.TLSConfig{
|
||||
CAPath: os.Getenv(vaultCAPath),
|
||||
}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := vault.NewClient(vconfig)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
accessToken, leaseDuration, err := getVaultAccessToken(c, config.Auth.AppRole.ID, config.Auth.AppRole.Secret)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// authenticate and get the access token
|
||||
c.SetToken(accessToken)
|
||||
v := vaultService{client: c, config: &config, leaseDuration: time.Duration(leaseDuration)}
|
||||
v.renewToken(c)
|
||||
return &v, nil
|
||||
}
|
||||
|
||||
func (v *vaultService) renewToken(c *vault.Client) {
|
||||
// renewToken starts a new go-routine which renews
|
||||
// the vault authentication token periodically.
|
||||
func (v *vaultService) renewToken() {
|
||||
retryDelay := 1 * time.Minute
|
||||
go func() {
|
||||
for {
|
||||
s, err := c.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
|
||||
if err != nil {
|
||||
time.Sleep(retryDelay)
|
||||
continue
|
||||
@@ -218,48 +156,54 @@ func (v *vaultService) renewToken(c *vault.Client) {
|
||||
}()
|
||||
}
|
||||
|
||||
// Generates a random plain text key, sealed plain text key from
|
||||
// Vault. It returns the plaintext key and sealed plaintext key on success
|
||||
// GenerateKey returns a new plaintext key, generated by the KMS,
|
||||
// and a sealed version of this plaintext key encrypted using the
|
||||
// named key referenced by keyID. It also binds the generated key
|
||||
// cryptographically to the provided context.
|
||||
func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.genDataKeyEndpoint(keyID), payload)
|
||||
|
||||
if err1 != nil {
|
||||
return key, sealedKey, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, sealedKey, err
|
||||
}
|
||||
sealKey := s.Data["ciphertext"].(string)
|
||||
plainKey, err := base64.StdEncoding.DecodeString(s.Data["plaintext"].(string))
|
||||
if err != nil {
|
||||
return key, sealedKey, err1
|
||||
return key, sealedKey, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
return key, []byte(sealKey), nil
|
||||
}
|
||||
|
||||
// unsealKMSKey unseals the sealedKey using the Vault master key
|
||||
// referenced by the keyID. The plain text key is returned on success.
|
||||
// UnsealKey returns the decrypted sealedKey as plaintext key.
|
||||
// Therefore it sends the sealedKey to the KMS which decrypts
|
||||
// it using the named key referenced by keyID and responses with
|
||||
// the plaintext key.
|
||||
//
|
||||
// The context must be same context as the one provided while
|
||||
// generating the plaintext key / sealedKey.
|
||||
func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
|
||||
contextStream := new(bytes.Buffer)
|
||||
ctx.WriteTo(contextStream)
|
||||
var contextStream bytes.Buffer
|
||||
ctx.WriteTo(&contextStream)
|
||||
|
||||
payload := map[string]interface{}{
|
||||
"ciphertext": string(sealedKey),
|
||||
"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
|
||||
}
|
||||
s, err1 := v.client.Logical().Write(v.decryptEndpoint(keyID), payload)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
base64Key := s.Data["plaintext"].(string)
|
||||
plainKey, err1 := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err1 != nil {
|
||||
return key, err1
|
||||
plainKey, err := base64.StdEncoding.DecodeString(base64Key)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
copy(key[:], []byte(plainKey))
|
||||
|
||||
return key, nil
|
||||
}
|
||||
|
||||
+1
-1
@@ -303,7 +303,7 @@ func (c cacheObjects) GetObject(ctx context.Context, bucket, object string, star
|
||||
}
|
||||
dcache.Delete(ctx, bucket, object)
|
||||
}
|
||||
if startOffset != 0 || length != objInfo.Size {
|
||||
if startOffset != 0 || (length > 0 && length != objInfo.Size) {
|
||||
// We don't cache partial objects.
|
||||
return GetObjectFn(ctx, bucket, object, startOffset, length, writer, etag, opts)
|
||||
}
|
||||
|
||||
@@ -154,6 +154,9 @@ func (api *DummyObjectLayer) DeleteBucketPolicy(context.Context, string) (err er
|
||||
func (api *DummyObjectLayer) IsNotificationSupported() (b bool) {
|
||||
return
|
||||
}
|
||||
func (api *DummyObjectLayer) IsListenBucketSupported() (b bool) {
|
||||
return
|
||||
}
|
||||
|
||||
func (api *DummyObjectLayer) IsEncryptionSupported() (b bool) {
|
||||
return
|
||||
|
||||
@@ -226,6 +226,17 @@ func TestGetSetIndexes(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func getHexSequences(start int, number int, paddinglen int) (seq []string) {
|
||||
for i := start; i <= number; i++ {
|
||||
if paddinglen == 0 {
|
||||
seq = append(seq, fmt.Sprintf("%x", i))
|
||||
} else {
|
||||
seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", paddinglen), i))
|
||||
}
|
||||
}
|
||||
return seq
|
||||
}
|
||||
|
||||
func getSequences(start int, number int, paddinglen int) (seq []string) {
|
||||
for i := start; i <= number; i++ {
|
||||
if paddinglen == 0 {
|
||||
@@ -472,6 +483,52 @@ func TestParseEndpointSet(t *testing.T) {
|
||||
},
|
||||
true,
|
||||
},
|
||||
// IPv6 ellipses with hexadecimal expansion
|
||||
{
|
||||
"http://[2001:3984:3989::{1...a}]/disk{1...10}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
Prefix: "http://[2001:3984:3989::",
|
||||
Suffix: "]/disk",
|
||||
Seq: getHexSequences(1, 10, 0),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
// IPv6 ellipses with hexadecimal expansion with 3 position numerics.
|
||||
{
|
||||
"http://[2001:3984:3989::{001...00a}]/disk{1...10}",
|
||||
endpointSet{
|
||||
[]ellipses.ArgPattern{
|
||||
[]ellipses.Pattern{
|
||||
{
|
||||
Prefix: "",
|
||||
Suffix: "",
|
||||
Seq: getSequences(1, 10, 0),
|
||||
},
|
||||
{
|
||||
Prefix: "http://[2001:3984:3989::",
|
||||
Suffix: "]/disk",
|
||||
Seq: getHexSequences(1, 10, 3),
|
||||
},
|
||||
},
|
||||
},
|
||||
nil,
|
||||
[][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}},
|
||||
},
|
||||
true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
|
||||
+30
-23
@@ -112,6 +112,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
|
||||
}
|
||||
}
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
|
||||
if host == "" {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: empty host name")
|
||||
@@ -152,7 +155,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
// Only check if the arg is an ip address and ask for scheme since its absent.
|
||||
// localhost, example.com, any FQDN cannot be disambiguated from a regular file path such as
|
||||
// /mnt/export1. So we go ahead and start the minio server in FS modes in these cases.
|
||||
if isHostIPv4(arg) {
|
||||
if isHostIP(arg) {
|
||||
return ep, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")
|
||||
}
|
||||
u = &url.URL{Path: path.Clean(arg)}
|
||||
@@ -224,7 +227,6 @@ func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
|
||||
uniqueArgs.Add(arg)
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
@@ -341,7 +343,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
if err != nil {
|
||||
host = endpoint.Host
|
||||
}
|
||||
hostIPSet, _ := getHostIP4(host)
|
||||
hostIPSet, _ := getHostIP(host)
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return serverAddr, endpoints, setupType,
|
||||
@@ -411,12 +413,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
host = localServerAddr
|
||||
}
|
||||
|
||||
ipList, err := getHostIP4(host)
|
||||
ipList, err := getHostIP(host)
|
||||
logger.FatalIf(err, "unexpected error when resolving host '%s'", host)
|
||||
|
||||
// Filter ipList by IPs those start with '127.'.
|
||||
// Filter ipList by IPs those start with '127.' or '::1'
|
||||
loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return strings.HasPrefix(ip, "127.")
|
||||
return strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
|
||||
}, "")
|
||||
|
||||
// If loop back IP is found and ipList contains only loop back IPs, then error out.
|
||||
@@ -455,7 +457,12 @@ func CreateEndpoints(serverAddr string, args ...[]string) (string, EndpointList,
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
updateDomainIPs(uniqueArgs)
|
||||
_, dok := os.LookupEnv("MINIO_DOMAIN")
|
||||
_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
|
||||
_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if dok && eok && !iok {
|
||||
updateDomainIPs(uniqueArgs)
|
||||
}
|
||||
|
||||
setupType = DistXLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
@@ -480,10 +487,10 @@ func GetLocalPeer(endpoints EndpointList) (localPeer string) {
|
||||
// Local peer can be empty in FS or Erasure coded mode.
|
||||
// If so, return globalMinioHost + globalMinioPort value.
|
||||
if globalMinioHost != "" {
|
||||
return globalMinioHost + ":" + globalMinioPort
|
||||
return net.JoinHostPort(globalMinioHost, globalMinioPort)
|
||||
}
|
||||
|
||||
return "127.0.0.1:" + globalMinioPort
|
||||
return net.JoinHostPort("127.0.0.1", globalMinioPort)
|
||||
}
|
||||
return peerSet.ToSlice()[0]
|
||||
}
|
||||
@@ -509,21 +516,21 @@ func GetRemotePeers(endpoints EndpointList) []string {
|
||||
return peerSet.ToSlice()
|
||||
}
|
||||
|
||||
// In federated and distributed setup, update IP addresses of the hosts passed in command line
|
||||
// if MINIO_PUBLIC_IPS are not set manually
|
||||
func updateDomainIPs(endPoints set.StringSet) {
|
||||
_, dok := os.LookupEnv("MINIO_DOMAIN")
|
||||
_, eok := os.LookupEnv("MINIO_ETCD_ENDPOINTS")
|
||||
_, iok := os.LookupEnv("MINIO_PUBLIC_IPS")
|
||||
if dok && eok && !iok {
|
||||
globalDomainIPs = set.NewStringSet()
|
||||
for e := range endPoints {
|
||||
host, _, _ := net.SplitHostPort(e)
|
||||
ipList, _ := getHostIP4(host)
|
||||
remoteIPList := ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.")
|
||||
}, "")
|
||||
globalDomainIPs.Add(remoteIPList.ToSlice()[0])
|
||||
ipList := set.NewStringSet()
|
||||
for e := range endPoints {
|
||||
host, _, err := net.SplitHostPort(e)
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "missing port in address") {
|
||||
host = e
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
}
|
||||
IPs, _ := getHostIP(host)
|
||||
ipList = ipList.Union(IPs)
|
||||
}
|
||||
globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.") || strings.HasPrefix(ip, "::1")
|
||||
}, "")
|
||||
}
|
||||
|
||||
@@ -304,27 +304,27 @@ func TestCreateEndpoints(t *testing.T) {
|
||||
}, DistXLSetupType, nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
for i, testCase := range testCases {
|
||||
serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
|
||||
|
||||
if err == nil {
|
||||
if testCase.expectedErr != nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
t.Fatalf("Test (%d) error: expected = %v, got = <nil>", i+1, testCase.expectedErr)
|
||||
} else {
|
||||
if serverAddr != testCase.expectedServerAddr {
|
||||
t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
|
||||
t.Fatalf("Test (%d) serverAddr: expected = %v, got = %v", i+1, testCase.expectedServerAddr, serverAddr)
|
||||
}
|
||||
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
|
||||
t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
|
||||
t.Fatalf("Test (%d) endpoints: expected = %v, got = %v", i+1, testCase.expectedEndpoints, endpoints)
|
||||
}
|
||||
if setupType != testCase.expectedSetupType {
|
||||
t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
|
||||
t.Fatalf("Test (%d) setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType)
|
||||
}
|
||||
}
|
||||
} else if testCase.expectedErr == nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
t.Fatalf("Test (%d) error: expected = <nil>, got = %v", i+1, err)
|
||||
} else if err.Error() != testCase.expectedErr.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
t.Fatalf("Test (%d) error: expected = %v, got = %v", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
// Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvKMSMasterKey is the environment variable used to specify
|
||||
// a KMS master key used to protect SSE-S3 per-object keys.
|
||||
// Valid values must be of the from: "KEY_ID:32_BYTE_HEX_VALUE".
|
||||
EnvKMSMasterKey = "MINIO_SSE_MASTER_KEY"
|
||||
|
||||
// EnvAutoEncryption is the environment variable used to en/disable
|
||||
// SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled,
|
||||
// requires a valid KMS configuration and turns any non-SSE-C
|
||||
// request into an SSE-S3 request.
|
||||
// If present EnvAutoEncryption must be either "on" or "off".
|
||||
EnvAutoEncryption = "MINIO_SSE_AUTO_ENCRYPTION"
|
||||
)
|
||||
|
||||
const (
|
||||
// EnvVaultEndpoint is the environment variable used to specify
|
||||
// the vault HTTPS endpoint.
|
||||
EnvVaultEndpoint = "MINIO_SSE_VAULT_ENDPOINT"
|
||||
|
||||
// EnvVaultAuthType is the environment variable used to specify
|
||||
// the authentication type for vault.
|
||||
EnvVaultAuthType = "MINIO_SSE_VAULT_AUTH_TYPE"
|
||||
|
||||
// EnvVaultAppRoleID is the environment variable used to specify
|
||||
// the vault AppRole ID.
|
||||
EnvVaultAppRoleID = "MINIO_SSE_VAULT_APPROLE_ID"
|
||||
|
||||
// EnvVaultAppSecretID is the environment variable used to specify
|
||||
// the vault AppRole secret corresponding to the AppRole ID.
|
||||
EnvVaultAppSecretID = "MINIO_SSE_VAULT_APPROLE_SECRET"
|
||||
|
||||
// EnvVaultKeyVersion is the environment variable used to specify
|
||||
// the vault key version.
|
||||
EnvVaultKeyVersion = "MINIO_SSE_VAULT_KEY_VERSION"
|
||||
|
||||
// EnvVaultKeyName is the environment variable used to specify
|
||||
// the vault named key-ring. In the S3 context it's referred as
|
||||
// customer master key ID (CMK-ID).
|
||||
EnvVaultKeyName = "MINIO_SSE_VAULT_KEY_NAME"
|
||||
|
||||
// EnvVaultCAPath is the environment variable used to specify the
|
||||
// path to a directory of PEM-encoded CA cert files. These CA cert
|
||||
// files are used to authenticate Minio to Vault over mTLS.
|
||||
EnvVaultCAPath = "MINIO_SSE_VAULT_CAPATH"
|
||||
|
||||
// EnvVaultNamespace is the environment variable used to specify
|
||||
// vault namespace. The vault namespace is used if the enterprise
|
||||
// version of Hashicorp Vault is used.
|
||||
EnvVaultNamespace = "MINIO_SSE_VAULT_NAMESPACE"
|
||||
)
|
||||
|
||||
// Environment provides functions for accessing environment
|
||||
// variables.
|
||||
var Environment = environment{}
|
||||
|
||||
type environment struct{}
|
||||
|
||||
// Get retrieves the value of the environment variable named
|
||||
// by the key. If the variable is present in the environment the
|
||||
// value (which may be empty) is returned. Otherwise it returns
|
||||
// the specified default value.
|
||||
func (environment) Get(key, defaultValue string) string {
|
||||
if v, ok := os.LookupEnv(key); ok {
|
||||
return v
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
// Lookup retrieves the value of the environment variable named
|
||||
// by the key. If the variable is present in the environment the
|
||||
// value (which may be empty) is returned and the boolean is true.
|
||||
// Otherwise the returned value will be empty and the boolean will
|
||||
// be false.
|
||||
func (environment) Lookup(key string) (string, bool) { return os.LookupEnv(key) }
|
||||
|
||||
// LookupKMSConfig extracts the KMS configuration provided by environment
|
||||
// variables and merge them with the provided KMS configuration. The
|
||||
// merging follows the following rules:
|
||||
//
|
||||
// 1. A valid value provided as environment variable is higher prioritized
|
||||
// than the provided configuration and overwrites the value from the
|
||||
// configuration file.
|
||||
//
|
||||
// 2. A value specified as environment variable never changes the configuration
|
||||
// file. So it is never made a persistent setting.
|
||||
//
|
||||
// It sets the global KMS configuration according to the merged configuration
|
||||
// on success.
|
||||
func (env environment) LookupKMSConfig(config crypto.KMSConfig) (err error) {
|
||||
// Lookup Hashicorp-Vault configuration & overwrite config entry if ENV var is present
|
||||
config.Vault.Endpoint = env.Get(EnvVaultEndpoint, config.Vault.Endpoint)
|
||||
config.Vault.CAPath = env.Get(EnvVaultCAPath, config.Vault.CAPath)
|
||||
config.Vault.Auth.Type = env.Get(EnvVaultAuthType, config.Vault.Auth.Type)
|
||||
config.Vault.Auth.AppRole.ID = env.Get(EnvVaultAppRoleID, config.Vault.Auth.AppRole.ID)
|
||||
config.Vault.Auth.AppRole.Secret = env.Get(EnvVaultAppSecretID, config.Vault.Auth.AppRole.Secret)
|
||||
config.Vault.Key.Name = env.Get(EnvVaultKeyName, config.Vault.Key.Name)
|
||||
config.Vault.Namespace = env.Get(EnvVaultNamespace, config.Vault.Namespace)
|
||||
keyVersion := env.Get(EnvVaultKeyVersion, strconv.Itoa(config.Vault.Key.Version))
|
||||
config.Vault.Key.Version, err = strconv.Atoi(keyVersion)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Invalid ENV variable: Unable to parse %s value (`%s`)", EnvVaultKeyVersion, keyVersion)
|
||||
}
|
||||
if err = config.Vault.Verify(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Lookup KMS master keys - only available through ENV.
|
||||
if masterKey, ok := env.Lookup(EnvKMSMasterKey); ok {
|
||||
if !config.Vault.IsEmpty() { // Vault and KMS master key provided
|
||||
return errors.New("Ambiguous KMS configuration: vault configuration and a master key are provided at the same time")
|
||||
}
|
||||
globalKMSKeyID, globalKMS, err = parseKMSMasterKey(masterKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if !config.Vault.IsEmpty() {
|
||||
globalKMS, err = crypto.NewVault(config.Vault)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalKMSKeyID = config.Vault.Key.Name
|
||||
}
|
||||
|
||||
autoEncryption, err := ParseBoolFlag(env.Get(EnvAutoEncryption, "off"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
globalAutoEncryption = bool(autoEncryption)
|
||||
if globalAutoEncryption && globalKMS == nil { // auto-encryption enabled but no KMS
|
||||
return errors.New("Invalid KMS configuration: auto-encryption is enabled but no valid KMS configuration is present")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// parseKMSMasterKey parses the value of the environment variable
|
||||
// `EnvKMSMasterKey` and returns a key-ID and a master-key KMS on success.
|
||||
func parseKMSMasterKey(envArg string) (string, crypto.KMS, error) {
|
||||
values := strings.SplitN(envArg, ":", 2)
|
||||
if len(values) != 2 {
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s does not contain a ':'", envArg)
|
||||
}
|
||||
var (
|
||||
keyID = values[0]
|
||||
hexKey = values[1]
|
||||
)
|
||||
if len(hexKey) != 64 { // 2 hex bytes = 1 byte
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
|
||||
}
|
||||
var masterKey [32]byte
|
||||
if _, err := hex.Decode(masterKey[:], []byte(hexKey)); err != nil {
|
||||
return "", nil, fmt.Errorf("Invalid KMS master key: %s not a 32 bytes long HEX value", hexKey)
|
||||
}
|
||||
return keyID, crypto.NewKMS(masterKey), nil
|
||||
}
|
||||
+1
-2
@@ -24,7 +24,6 @@ import (
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
@@ -273,7 +272,7 @@ func initFormatFS(ctx context.Context, fsPath string) (rlk *lock.RLockedFile, er
|
||||
rlk.Close()
|
||||
return nil, err
|
||||
}
|
||||
logger.SetDeploymentID(id)
|
||||
globalDeploymentID = id
|
||||
return rlk, nil
|
||||
}
|
||||
}
|
||||
|
||||
@@ -381,6 +381,7 @@ func saveFormatXL(disk StorageAPI, format interface{}) error {
|
||||
|
||||
var ignoredHiddenDirectories = []string{
|
||||
minioMetaBucket,
|
||||
".snapshot",
|
||||
"lost+found",
|
||||
"$RECYCLE.BIN",
|
||||
"System Volume Information",
|
||||
|
||||
@@ -682,6 +682,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
fsRemoveAll(ctx, uploadIDDir)
|
||||
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
||||
fsRemoveDir(ctx, fs.getMultipartSHADir(bucket, object))
|
||||
fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
@@ -769,6 +771,9 @@ func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupIn
|
||||
}
|
||||
if now.Sub(fi.ModTime()) > expiry {
|
||||
fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry, uploadID))
|
||||
// It is safe to ignore any directory not empty error (in case there were multiple uploadIDs on the same object)
|
||||
fsRemoveDir(ctx, pathJoin(fs.fsPath, minioMetaMultipartBucket, entry))
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+9
-14
@@ -187,14 +187,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
case <-doneCh:
|
||||
return errWalkAbort
|
||||
default:
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
}
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
atomic.AddUint64(&fs.totalUsed, uint64(fi.Size()))
|
||||
@@ -226,14 +221,9 @@ func (fs *FSObjects) diskUsage(doneCh chan struct{}) {
|
||||
}
|
||||
}
|
||||
|
||||
var fi os.FileInfo
|
||||
var err error
|
||||
if hasSuffix(entry, slashSeparator) {
|
||||
fi, err = fsStatDir(ctx, entry)
|
||||
} else {
|
||||
fi, err = fsStatFile(ctx, entry)
|
||||
}
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
usage = usage + uint64(fi.Size())
|
||||
@@ -1316,6 +1306,11 @@ func (fs *FSObjects) IsNotificationSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (fs *FSObjects) IsListenBucketSupported() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
func (fs *FSObjects) IsEncryptionSupported() bool {
|
||||
return true
|
||||
|
||||
+54
-50
@@ -18,8 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/signal"
|
||||
@@ -41,7 +41,7 @@ func init() {
|
||||
var (
|
||||
gatewayCmd = cli.Command{
|
||||
Name: "gateway",
|
||||
Usage: "Start object storage gateway.",
|
||||
Usage: "start object storage gateway",
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
@@ -112,39 +112,17 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if jsonFlag {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if quietFlag {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Fetch address option
|
||||
gatewayAddr := ctx.GlobalString("address")
|
||||
if gatewayAddr == ":"+globalMinioPort {
|
||||
gatewayAddr = ctx.String("address")
|
||||
}
|
||||
|
||||
// Handle common command args.
|
||||
handleCommonCmdArgs(ctx)
|
||||
|
||||
// Get port to listen on from gateway address
|
||||
_, gatewayPort, pErr := net.SplitHostPort(gatewayAddr)
|
||||
if pErr != nil {
|
||||
logger.FatalIf(pErr, "Unable to start gateway")
|
||||
}
|
||||
globalMinioHost, globalMinioPort = mustSplitHostPort(globalCLIContext.Addr)
|
||||
|
||||
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
|
||||
// to IPv6 address ie minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error situation we check for port availability.
|
||||
logger.FatalIf(checkPortAvailability(gatewayPort), "Unable to start the gateway")
|
||||
logger.FatalIf(checkPortAvailability(globalMinioPort), "Unable to start the gateway")
|
||||
|
||||
// Create certs path.
|
||||
logger.FatalIf(createConfigDir(), "Unable to create configuration directories")
|
||||
@@ -176,11 +154,12 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
if globalEtcdClient != nil {
|
||||
// Enable STS router if etcd is enabled.
|
||||
registerSTSRouter(router)
|
||||
|
||||
// Enable admin router if etcd is enabled.
|
||||
registerAdminRouter(router)
|
||||
}
|
||||
|
||||
// Enable IAM admin APIs if etcd is enabled, if not just enable basic
|
||||
// operations such as profiling, server info etc.
|
||||
registerAdminRouter(router, globalEtcdClient != nil)
|
||||
|
||||
// Add healthcheck router
|
||||
registerHealthCheckRouter(router)
|
||||
|
||||
@@ -195,12 +174,21 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
// Add API router.
|
||||
registerAPIRouter(router)
|
||||
|
||||
// Dummy endpoint representing gateway instance.
|
||||
globalEndpoints = []Endpoint{{
|
||||
URL: &url.URL{Path: "/minio/gateway"},
|
||||
IsLocal: true,
|
||||
}}
|
||||
|
||||
// Initialize Admin Peers.
|
||||
initGlobalAdminPeers(globalEndpoints)
|
||||
|
||||
var getCert certs.GetCertificateFunc
|
||||
if globalTLSCerts != nil {
|
||||
getCert = globalTLSCerts.GetCertificate
|
||||
}
|
||||
|
||||
globalHTTPServer = xhttp.NewServer([]string{gatewayAddr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
|
||||
globalHTTPServer = xhttp.NewServer([]string{globalCLIContext.Addr}, criticalErrorHandler{registerHandlers(router, globalHandlers...)}, getCert)
|
||||
globalHTTPServer.UpdateBytesReadFunc = globalConnStats.incInputBytes
|
||||
globalHTTPServer.UpdateBytesWrittenFunc = globalConnStats.incOutputBytes
|
||||
go func() {
|
||||
@@ -209,6 +197,22 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
|
||||
|
||||
// !!! Do not move this block !!!
|
||||
// For all gateways, the config needs to be loaded from env
|
||||
// prior to initializing the gateway layer
|
||||
{
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// Override any values from ENVs.
|
||||
srvCfg.loadFromEnvs()
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
}
|
||||
|
||||
newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
|
||||
if err != nil {
|
||||
// Stop watching for any certificate changes.
|
||||
@@ -218,28 +222,18 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
logger.FatalIf(err, "Unable to initialize gateway backend")
|
||||
}
|
||||
|
||||
// Create a new config system.
|
||||
globalConfigSys = NewConfigSys()
|
||||
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// Override any values from ENVs.
|
||||
srvCfg.loadFromEnvs()
|
||||
|
||||
// Load values to cached global values.
|
||||
srvCfg.loadToCachedConfigs()
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
if globalEtcdClient != nil && gatewayName == "nas" {
|
||||
// Create a new config system.
|
||||
globalConfigSys = NewConfigSys()
|
||||
|
||||
// Load globalServerConfig from etcd
|
||||
_ = globalConfigSys.Init(newObject)
|
||||
}
|
||||
// Load logger subsystem
|
||||
loadLoggers()
|
||||
|
||||
// This is only to uniquely identify each gateway deployments.
|
||||
logger.SetDeploymentID(os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID"))
|
||||
globalDeploymentID = os.Getenv("MINIO_GATEWAY_DEPLOYMENT_ID")
|
||||
|
||||
var cacheConfig = globalServerConfig.GetCacheConfig()
|
||||
if len(cacheConfig.Drives) > 0 {
|
||||
@@ -256,7 +250,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
globalIAMSys = NewIAMSys()
|
||||
if globalEtcdClient != nil {
|
||||
// Initialize IAM sys.
|
||||
go globalIAMSys.Init(newObject)
|
||||
_ = globalIAMSys.Init(newObject)
|
||||
}
|
||||
|
||||
// Create new policy system.
|
||||
@@ -267,6 +261,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
// Create new notification system.
|
||||
globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
|
||||
if globalEtcdClient != nil && newObject.IsNotificationSupported() {
|
||||
_ = globalNotificationSys.Init(newObject)
|
||||
}
|
||||
|
||||
if globalAutoEncryption && !newObject.IsEncryptionSupported() {
|
||||
logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but gateway does not support encryption")
|
||||
}
|
||||
|
||||
// Once endpoints are finalized, initialize the new object api.
|
||||
globalObjLayerMutex.Lock()
|
||||
@@ -274,7 +275,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
if !quietFlag {
|
||||
if !globalCLIContext.Quiet {
|
||||
mode := globalMinioModeGatewayPrefix + gatewayName
|
||||
// Check update mode.
|
||||
checkUpdate(mode)
|
||||
@@ -285,8 +286,11 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
}
|
||||
|
||||
// Print gateway startup message.
|
||||
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
|
||||
printGatewayStartupMessage(getAPIEndpoints(), gatewayName)
|
||||
}
|
||||
|
||||
// Set uptime time after object layer has initialized.
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
handleSignals()
|
||||
}
|
||||
|
||||
@@ -55,12 +55,14 @@ func printGatewayCommonMsg(apiEndpoints []string) {
|
||||
cred := globalServerConfig.GetCredential()
|
||||
|
||||
apiEndpointStr := strings.Join(apiEndpoints, " ")
|
||||
|
||||
// Colorize the message and print.
|
||||
logger.StartupMessage(colorBlue("\nEndpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
if isTerminal() {
|
||||
logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
if isTerminal() && !globalCLIContext.Anonymous {
|
||||
logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
}
|
||||
printEventNotifiers()
|
||||
|
||||
if globalIsBrowserEnabled {
|
||||
logger.StartupMessage(colorBlue("\nBrowser Access:"))
|
||||
|
||||
@@ -132,6 +132,11 @@ func (a GatewayUnsupported) IsNotificationSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this layer.
|
||||
func (a GatewayUnsupported) IsListenBucketSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncryptionSupported returns whether server side encryption is applicable for this layer.
|
||||
func (a GatewayUnsupported) IsEncryptionSupported() bool {
|
||||
return false
|
||||
|
||||
@@ -92,7 +92,7 @@ EXAMPLES:
|
||||
`
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: b2Backend,
|
||||
Usage: "Backblaze B2.",
|
||||
Usage: "Backblaze B2",
|
||||
Action: b2GatewayMain,
|
||||
CustomHelpTemplate: b2GatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
|
||||
@@ -147,7 +147,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: gcsBackend,
|
||||
Usage: "Google Cloud Storage.",
|
||||
Usage: "Google Cloud Storage",
|
||||
Action: gcsGatewayMain,
|
||||
CustomHelpTemplate: gcsGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -206,11 +206,10 @@ func (g *GCS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
|
||||
gcs := &gcsGateway{
|
||||
client: client,
|
||||
projectID: g.projectID,
|
||||
ctx: ctx,
|
||||
}
|
||||
|
||||
// Start background process to cleanup old files in minio.sys.tmp
|
||||
go gcs.CleanupGCSMinioSysTmp()
|
||||
go gcs.CleanupGCSMinioSysTmp(ctx)
|
||||
return gcs, nil
|
||||
}
|
||||
|
||||
@@ -349,7 +348,6 @@ type gcsGateway struct {
|
||||
minio.GatewayUnsupported
|
||||
client *storage.Client
|
||||
projectID string
|
||||
ctx context.Context
|
||||
}
|
||||
|
||||
// Returns projectID from the GOOGLE_APPLICATION_CREDENTIALS file.
|
||||
@@ -366,8 +364,8 @@ func gcsParseProjectID(credsFile string) (projectID string, err error) {
|
||||
}
|
||||
|
||||
// Cleanup old files in minio.sys.tmp of the given bucket.
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(ctx context.Context, bucket string) {
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: minio.GatewayMinioSysTmp, Versions: false})
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err != nil {
|
||||
@@ -380,7 +378,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
}
|
||||
if time.Since(attrs.Updated) > gcsMultipartExpiry {
|
||||
// Delete files older than 2 weeks.
|
||||
err := l.client.Bucket(bucket).Object(attrs.Name).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Object(attrs.Name).Delete(ctx)
|
||||
if err != nil {
|
||||
reqInfo := &logger.ReqInfo{BucketName: bucket, ObjectName: attrs.Name}
|
||||
ctx := logger.SetReqInfo(context.Background(), reqInfo)
|
||||
@@ -392,9 +390,9 @@ func (l *gcsGateway) CleanupGCSMinioSysTmpBucket(bucket string) {
|
||||
}
|
||||
|
||||
// Cleanup old files in minio.sys.tmp of all buckets.
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmp() {
|
||||
func (l *gcsGateway) CleanupGCSMinioSysTmp(ctx context.Context) {
|
||||
for {
|
||||
it := l.client.Buckets(l.ctx, l.projectID)
|
||||
it := l.client.Buckets(ctx, l.projectID)
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
if err != nil {
|
||||
@@ -404,7 +402,7 @@ func (l *gcsGateway) CleanupGCSMinioSysTmp() {
|
||||
}
|
||||
break
|
||||
}
|
||||
l.CleanupGCSMinioSysTmpBucket(attrs.Name)
|
||||
l.CleanupGCSMinioSysTmpBucket(ctx, attrs.Name)
|
||||
}
|
||||
// Run the cleanup loop every 1 day.
|
||||
time.Sleep(gcsCleanupInterval)
|
||||
@@ -431,7 +429,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
|
||||
location = "us"
|
||||
}
|
||||
|
||||
err := bkt.Create(l.ctx, l.projectID, &storage.BucketAttrs{
|
||||
err := bkt.Create(ctx, l.projectID, &storage.BucketAttrs{
|
||||
Location: location,
|
||||
})
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -440,7 +438,7 @@ func (l *gcsGateway) MakeBucketWithLocation(ctx context.Context, bucket, locatio
|
||||
|
||||
// GetBucketInfo - Get bucket metadata..
|
||||
func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.BucketInfo, error) {
|
||||
attrs, err := l.client.Bucket(bucket).Attrs(l.ctx)
|
||||
attrs, err := l.client.Bucket(bucket).Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.BucketInfo{}, gcsToObjectError(err, bucket)
|
||||
@@ -454,7 +452,7 @@ func (l *gcsGateway) GetBucketInfo(ctx context.Context, bucket string) (minio.Bu
|
||||
|
||||
// ListBuckets lists all buckets under your project-id on GCS.
|
||||
func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInfo, err error) {
|
||||
it := l.client.Buckets(l.ctx, l.projectID)
|
||||
it := l.client.Buckets(ctx, l.projectID)
|
||||
|
||||
// Iterate and capture all the buckets.
|
||||
for {
|
||||
@@ -479,7 +477,7 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
|
||||
|
||||
// DeleteBucket delete a bucket on GCS.
|
||||
func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
itObject := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
itObject := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: "/",
|
||||
Versions: false,
|
||||
})
|
||||
@@ -509,7 +507,7 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
}
|
||||
if gcsMinioPathFound {
|
||||
// Remove minio.sys.tmp before deleting the bucket.
|
||||
itObject = l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
|
||||
itObject = l.client.Bucket(bucket).Objects(ctx, &storage.Query{Versions: false, Prefix: minio.GatewayMinioSysTmp})
|
||||
for {
|
||||
objAttrs, err := itObject.Next()
|
||||
if err == iterator.Done {
|
||||
@@ -519,14 +517,14 @@ func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string) error {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err)
|
||||
}
|
||||
err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(l.ctx)
|
||||
err = l.client.Bucket(bucket).Object(objAttrs.Name).Delete(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
err := l.client.Bucket(bucket).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Delete(ctx)
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -561,7 +559,7 @@ func (l *gcsGateway) ListObjects(ctx context.Context, bucket string, prefix stri
|
||||
return minio.ListObjectsInfo{}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: delimiter,
|
||||
Prefix: prefix,
|
||||
Versions: false,
|
||||
@@ -668,7 +666,7 @@ func (l *gcsGateway) ListObjectsV2(ctx context.Context, bucket, prefix, continua
|
||||
return minio.ListObjectsV2Info{ContinuationToken: continuationToken}, nil
|
||||
}
|
||||
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
|
||||
Delimiter: delimiter,
|
||||
Prefix: prefix,
|
||||
Versions: false,
|
||||
@@ -769,7 +767,7 @@ func (l *gcsGateway) GetObjectNInfo(ctx context.Context, bucket, object string,
|
||||
func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, startOffset int64, length int64, writer io.Writer, etag string, opts minio.ObjectOptions) error {
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -781,7 +779,7 @@ func (l *gcsGateway) GetObject(ctx context.Context, bucket string, key string, s
|
||||
// Calling ReadCompressed with true accomplishes that.
|
||||
object := l.client.Bucket(bucket).Object(key).ReadCompressed(true)
|
||||
|
||||
r, err := object.NewRangeReader(l.ctx, startOffset, length)
|
||||
r, err := object.NewRangeReader(ctx, startOffset, length)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, key)
|
||||
@@ -864,12 +862,12 @@ func applyMetadataToGCSAttrs(metadata map[string]string, attrs *storage.ObjectAt
|
||||
func (l *gcsGateway) GetObjectInfo(ctx context.Context, bucket string, object string, opts minio.ObjectOptions) (minio.ObjectInfo, error) {
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
attrs, err := l.client.Bucket(bucket).Object(object).Attrs(l.ctx)
|
||||
attrs, err := l.client.Bucket(bucket).Object(object).Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, object)
|
||||
@@ -884,14 +882,14 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
|
||||
|
||||
// if we want to mimic S3 behavior exactly, we need to verify if bucket exists first,
|
||||
// otherwise gcs will just return object not exist in case of non-existing bucket
|
||||
if _, err := l.client.Bucket(bucket).Attrs(l.ctx); err != nil {
|
||||
if _, err := l.client.Bucket(bucket).Attrs(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
object := l.client.Bucket(bucket).Object(key)
|
||||
|
||||
w := object.NewWriter(l.ctx)
|
||||
w := object.NewWriter(ctx)
|
||||
// Disable "chunked" uploading in GCS client if the size of the data to be uploaded is below
|
||||
// the current chunk-size of the writer. This avoids an unnecessary memory allocation.
|
||||
if data.Size() < int64(w.ChunkSize) {
|
||||
@@ -909,7 +907,7 @@ func (l *gcsGateway) PutObject(ctx context.Context, bucket string, key string, r
|
||||
// Close the object writer upon success.
|
||||
w.Close()
|
||||
|
||||
attrs, err := object.Attrs(l.ctx)
|
||||
attrs, err := object.Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -928,7 +926,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
|
||||
copier := dst.CopierFrom(src)
|
||||
applyMetadataToGCSAttrs(srcInfo.UserDefined, &copier.ObjectAttrs)
|
||||
|
||||
attrs, err := copier.Run(l.ctx)
|
||||
attrs, err := copier.Run(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, destBucket, destObject)
|
||||
@@ -939,7 +937,7 @@ func (l *gcsGateway) CopyObject(ctx context.Context, srcBucket string, srcObject
|
||||
|
||||
// DeleteObject - Deletes a blob in bucket
|
||||
func (l *gcsGateway) DeleteObject(ctx context.Context, bucket string, object string) error {
|
||||
err := l.client.Bucket(bucket).Object(object).Delete(l.ctx)
|
||||
err := l.client.Bucket(bucket).Object(object).Delete(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, object)
|
||||
@@ -956,7 +954,7 @@ func (l *gcsGateway) NewMultipartUpload(ctx context.Context, bucket string, key
|
||||
// generate name for part zero
|
||||
meta := gcsMultipartMetaName(uploadID)
|
||||
|
||||
w := l.client.Bucket(bucket).Object(meta).NewWriter(l.ctx)
|
||||
w := l.client.Bucket(bucket).Object(meta).NewWriter(ctx)
|
||||
defer w.Close()
|
||||
|
||||
applyMetadataToGCSAttrs(metadata, &w.ObjectAttrs)
|
||||
@@ -1056,7 +1054,7 @@ func (l *gcsGateway) ListMultipartUploads(ctx context.Context, bucket string, pr
|
||||
// Checks if minio.sys.tmp/multipart/v1/<upload-id>/gcs.json exists, returns
|
||||
// an object layer compatible error upon any error.
|
||||
func (l *gcsGateway) checkUploadIDExists(ctx context.Context, bucket string, key string, uploadID string) error {
|
||||
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(l.ctx)
|
||||
_, err := l.client.Bucket(bucket).Object(gcsMultipartMetaName(uploadID)).Attrs(ctx)
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket, key, uploadID)
|
||||
}
|
||||
@@ -1073,7 +1071,7 @@ func (l *gcsGateway) PutObjectPart(ctx context.Context, bucket string, key strin
|
||||
etag = minio.GenETag()
|
||||
}
|
||||
object := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, partNumber, etag))
|
||||
w := object.NewWriter(l.ctx)
|
||||
w := object.NewWriter(ctx)
|
||||
// Disable "chunked" uploading in GCS client. If enabled, it can cause a corner case
|
||||
// where it tries to upload 0 bytes in the last chunk and get error from server.
|
||||
w.ChunkSize = 0
|
||||
@@ -1186,7 +1184,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
|
||||
prefix := fmt.Sprintf("%s/%s/", gcsMinioMultipartPathV1, uploadID)
|
||||
|
||||
// iterate through all parts and delete them
|
||||
it := l.client.Bucket(bucket).Objects(l.ctx, &storage.Query{Prefix: prefix, Versions: false})
|
||||
it := l.client.Bucket(bucket).Objects(ctx, &storage.Query{Prefix: prefix, Versions: false})
|
||||
|
||||
for {
|
||||
attrs, err := it.Next()
|
||||
@@ -1200,7 +1198,7 @@ func (l *gcsGateway) cleanupMultipartUpload(ctx context.Context, bucket, key, up
|
||||
|
||||
object := l.client.Bucket(bucket).Object(attrs.Name)
|
||||
// Ignore the error as parallel AbortMultipartUpload might have deleted it.
|
||||
object.Delete(l.ctx)
|
||||
object.Delete(ctx)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -1222,13 +1220,13 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
meta := gcsMultipartMetaName(uploadID)
|
||||
object := l.client.Bucket(bucket).Object(meta)
|
||||
|
||||
partZeroAttrs, err := object.Attrs(l.ctx)
|
||||
partZeroAttrs, err := object.Attrs(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key, uploadID)
|
||||
}
|
||||
|
||||
r, err := object.NewReader(l.ctx)
|
||||
r, err := object.NewReader(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -1259,7 +1257,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
for i, uploadedPart := range uploadedParts {
|
||||
parts = append(parts, l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID,
|
||||
uploadedPart.PartNumber, uploadedPart.ETag)))
|
||||
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(l.ctx)
|
||||
partAttr, pErr := l.client.Bucket(bucket).Object(gcsMultipartDataName(uploadID, uploadedPart.PartNumber, uploadedPart.ETag)).Attrs(ctx)
|
||||
if pErr != nil {
|
||||
logger.LogIf(ctx, pErr)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(pErr, bucket, key, uploadID)
|
||||
@@ -1305,7 +1303,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
composer.ContentType = partZeroAttrs.ContentType
|
||||
composer.Metadata = partZeroAttrs.Metadata
|
||||
|
||||
if _, err = composer.Run(l.ctx); err != nil {
|
||||
if _, err = composer.Run(ctx); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
}
|
||||
@@ -1322,7 +1320,7 @@ func (l *gcsGateway) CompleteMultipartUpload(ctx context.Context, bucket string,
|
||||
composer.ContentDisposition = partZeroAttrs.ContentDisposition
|
||||
composer.ContentLanguage = partZeroAttrs.ContentLanguage
|
||||
composer.Metadata = partZeroAttrs.Metadata
|
||||
attrs, err := composer.Run(l.ctx)
|
||||
attrs, err := composer.Run(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return minio.ObjectInfo{}, gcsToObjectError(err, bucket, key)
|
||||
@@ -1362,7 +1360,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
|
||||
acl := l.client.Bucket(bucket).ACL()
|
||||
if policies[0].Policy == miniogopolicy.BucketPolicyNone {
|
||||
if err := acl.Delete(l.ctx, storage.AllUsers); err != nil {
|
||||
if err := acl.Delete(ctx, storage.AllUsers); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1380,7 +1378,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
return minio.NotImplemented{}
|
||||
}
|
||||
|
||||
if err := acl.Set(l.ctx, storage.AllUsers, role); err != nil {
|
||||
if err := acl.Set(ctx, storage.AllUsers, role); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1390,7 +1388,7 @@ func (l *gcsGateway) SetBucketPolicy(ctx context.Context, bucket string, bucketP
|
||||
|
||||
// GetBucketPolicy - Get policy on bucket
|
||||
func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*policy.Policy, error) {
|
||||
rules, err := l.client.Bucket(bucket).ACL().List(l.ctx)
|
||||
rules, err := l.client.Bucket(bucket).ACL().List(ctx)
|
||||
if err != nil {
|
||||
return nil, gcsToObjectError(err, bucket)
|
||||
}
|
||||
@@ -1449,7 +1447,7 @@ func (l *gcsGateway) GetBucketPolicy(ctx context.Context, bucket string) (*polic
|
||||
// DeleteBucketPolicy - Delete all policies on bucket
|
||||
func (l *gcsGateway) DeleteBucketPolicy(ctx context.Context, bucket string) error {
|
||||
// This only removes the storage.AllUsers policies
|
||||
if err := l.client.Bucket(bucket).ACL().Delete(l.ctx, storage.AllUsers); err != nil {
|
||||
if err := l.client.Bucket(bucket).ACL().Delete(ctx, storage.AllUsers); err != nil {
|
||||
return gcsToObjectError(err, bucket)
|
||||
}
|
||||
|
||||
|
||||
@@ -108,7 +108,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: mantaBackend,
|
||||
Usage: "Manta Object Storage.",
|
||||
Usage: "Manta Object Storage",
|
||||
Action: mantaGatewayMain,
|
||||
CustomHelpTemplate: mantaGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
|
||||
@@ -74,7 +74,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: nasBackend,
|
||||
Usage: "Network-attached storage (NAS).",
|
||||
Usage: "Network-attached storage (NAS)",
|
||||
Action: nasGatewayMain,
|
||||
CustomHelpTemplate: nasGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -108,7 +108,7 @@ func (g *NAS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &nasObjects{newObject.(*minio.FSObjects)}, nil
|
||||
return &nasObjects{newObject}, nil
|
||||
}
|
||||
|
||||
// Production - nas gateway is production ready.
|
||||
@@ -116,17 +116,12 @@ func (g *NAS) Production() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// IsListenBucketSupported returns whether listen bucket notification is applicable for this gateway.
|
||||
func (n *nasObjects) IsListenBucketSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// nasObjects implements gateway for Minio and S3 compatible object storage servers.
|
||||
type nasObjects struct {
|
||||
*minio.FSObjects
|
||||
}
|
||||
|
||||
// IsNotificationSupported returns whether notifications are applicable for this layer.
|
||||
func (l *nasObjects) IsNotificationSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// IsCompressionSupported returns whether compression is applicable for this layer.
|
||||
func (l *nasObjects) IsCompressionSupported() bool {
|
||||
return false
|
||||
minio.ObjectLayer
|
||||
}
|
||||
|
||||
@@ -98,7 +98,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: "oss",
|
||||
Usage: "Alibaba Cloud (Aliyun) Object Storage Service (OSS).",
|
||||
Usage: "Alibaba Cloud (Aliyun) Object Storage Service (OSS)",
|
||||
Action: ossGatewayMain,
|
||||
CustomHelpTemplate: ossGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
|
||||
@@ -111,7 +111,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: s3Backend,
|
||||
Usage: "Amazon Simple Storage Service (S3).",
|
||||
Usage: "Amazon Simple Storage Service (S3)",
|
||||
Action: s3GatewayMain,
|
||||
CustomHelpTemplate: s3GatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
@@ -510,12 +510,12 @@ func (l *s3Objects) AbortMultipartUpload(ctx context.Context, bucket string, obj
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (l *s3Objects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, uploadedParts []minio.CompletePart, opts minio.ObjectOptions) (oi minio.ObjectInfo, e error) {
|
||||
err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
|
||||
etag, err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, minio.ToMinioClientCompleteParts(uploadedParts))
|
||||
if err != nil {
|
||||
return oi, minio.ErrorRespToObjectError(err, bucket, object)
|
||||
}
|
||||
|
||||
return l.GetObjectInfo(ctx, bucket, object, minio.ObjectOptions{})
|
||||
return minio.ObjectInfo{Bucket: bucket, Name: object, ETag: etag}, nil
|
||||
}
|
||||
|
||||
// SetBucketPolicy sets policy on bucket
|
||||
|
||||
@@ -97,7 +97,7 @@ EXAMPLES:
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: siaBackend,
|
||||
Usage: "Sia Decentralized Cloud.",
|
||||
Usage: "Sia Decentralized Cloud",
|
||||
Action: siaGatewayMain,
|
||||
CustomHelpTemplate: siaGatewayTemplate,
|
||||
HideHelpCommand: true,
|
||||
|
||||
+77
-32
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
@@ -91,7 +90,7 @@ func setRequestHeaderSizeLimitHandler(h http.Handler) http.Handler {
|
||||
// of the user-defined metadata to 2 KB.
|
||||
func (h requestHeaderSizeLimitHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if isHTTPHeaderSizeTooLarge(r.Header) {
|
||||
writeErrorResponse(w, ErrMetadataTooLarge, r.URL)
|
||||
writeErrorResponse(w, ErrMetadataTooLarge, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
h.Handler.ServeHTTP(w, r)
|
||||
@@ -134,7 +133,7 @@ func filterReservedMetadata(h http.Handler) http.Handler {
|
||||
// would be treated as metadata.
|
||||
func (h reservedMetadataHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if containsReservedMetadata(r.Header) {
|
||||
writeErrorResponse(w, ErrUnsupportedMetadata, r.URL)
|
||||
writeErrorResponse(w, ErrUnsupportedMetadata, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
h.Handler.ServeHTTP(w, r)
|
||||
@@ -228,7 +227,8 @@ func guessIsRPCReq(req *http.Request) bool {
|
||||
if req == nil {
|
||||
return false
|
||||
}
|
||||
return req.Method == http.MethodPost
|
||||
return req.Method == http.MethodPost &&
|
||||
strings.HasPrefix(req.URL.Path, minioReservedBucketPath+"/")
|
||||
}
|
||||
|
||||
func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -300,7 +300,7 @@ func (h minioReservedBucketHandler) ServeHTTP(w http.ResponseWriter, r *http.Req
|
||||
// buckets
|
||||
bucketName, _ := urlPath2BucketObjectName(r.URL.Path)
|
||||
if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) {
|
||||
writeErrorResponse(w, ErrAllAccessDisabled, r.URL)
|
||||
writeErrorResponse(w, ErrAllAccessDisabled, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -363,14 +363,14 @@ func (h timeValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// All our internal APIs are sensitive towards Date
|
||||
// header, for all requests where Date header is not
|
||||
// present we will reject such clients.
|
||||
writeErrorResponse(w, apiErr, r.URL)
|
||||
writeErrorResponse(w, apiErr, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past
|
||||
// or in the future, reject request otherwise.
|
||||
curTime := UTCNow()
|
||||
if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime {
|
||||
writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL)
|
||||
writeErrorResponse(w, ErrRequestTimeTooSkewed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -485,20 +485,20 @@ func (h resourceHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// If bucketName is present and not objectName check for bucket level resource queries.
|
||||
if bucketName != "" && objectName == "" {
|
||||
if ignoreNotImplementedBucketResources(r) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
// If bucketName and objectName are present check for its resource queries.
|
||||
if bucketName != "" && objectName != "" {
|
||||
if ignoreNotImplementedObjectResources(r) {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
// A put method on path "/" doesn't make sense, ignore it.
|
||||
if r.Method == http.MethodPut && r.URL.Path == "/" {
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -601,14 +601,14 @@ func hasBadPathComponent(path string) bool {
|
||||
func (h pathValidityHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Check for bad components in URL path.
|
||||
if hasBadPathComponent(r.URL.Path) {
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
// Check for bad components in URL query values.
|
||||
for _, vv := range r.URL.Query() {
|
||||
for _, v := range vv {
|
||||
if hasBadPathComponent(v) {
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL)
|
||||
writeErrorResponse(w, ErrInvalidResourceName, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -625,10 +625,53 @@ type bucketForwardingHandler struct {
|
||||
}
|
||||
|
||||
func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if globalDNSConfig == nil || globalDomainName == "" || guessIsBrowserReq(r) || guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || guessIsRPCReq(r) {
|
||||
if globalDNSConfig == nil || globalDomainName == "" ||
|
||||
guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
|
||||
guessIsRPCReq(r) || isAdminReq(r) {
|
||||
f.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
// For browser requests, when federation is setup we need to
|
||||
// specifically handle download and upload for browser requests.
|
||||
if guessIsBrowserReq(r) && globalDNSConfig != nil && globalDomainName != "" {
|
||||
var bucket, _ string
|
||||
switch r.Method {
|
||||
case http.MethodPut:
|
||||
if getRequestAuthType(r) == authTypeJWT {
|
||||
bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/upload"))
|
||||
}
|
||||
case http.MethodGet:
|
||||
if t := r.URL.Query().Get("token"); t != "" {
|
||||
bucket, _ = urlPath2BucketObjectName(strings.TrimPrefix(r.URL.Path, minioReservedBucketPath+"/download"))
|
||||
} else if getRequestAuthType(r) != authTypeJWT && !strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
|
||||
bucket, _ = urlPath2BucketObjectName(r.URL.Path)
|
||||
}
|
||||
}
|
||||
if bucket != "" {
|
||||
sr, err := globalDNSConfig.Get(bucket)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
|
||||
} else {
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
r.URL.Scheme = "http"
|
||||
if globalIsSSL {
|
||||
r.URL.Scheme = "https"
|
||||
}
|
||||
r.URL.Host = getHostFromSrv(sr)
|
||||
f.fwd.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
f.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
|
||||
bucket, object := urlPath2BucketObjectName(r.URL.Path)
|
||||
// ListBucket requests should be handled at current endpoint as
|
||||
// all buckets data can be fetched from here.
|
||||
@@ -653,19 +696,18 @@ func (f bucketForwardingHandler) ServeHTTP(w http.ResponseWriter, r *http.Reques
|
||||
sr, err := globalDNSConfig.Get(bucket)
|
||||
if err != nil {
|
||||
if err == dns.ErrNoEntriesFound {
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL)
|
||||
writeErrorResponse(w, ErrNoSuchBucket, r.URL, guessIsBrowserReq(r))
|
||||
} else {
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL)
|
||||
writeErrorResponse(w, toAPIErrorCode(context.Background(), err), r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() {
|
||||
host, port := getRandomHostPort(sr)
|
||||
r.URL.Scheme = "http"
|
||||
if globalIsSSL {
|
||||
r.URL.Scheme = "https"
|
||||
}
|
||||
r.URL.Host = fmt.Sprintf("%s:%d", host, port)
|
||||
r.URL.Host = getHostFromSrv(sr)
|
||||
f.fwd.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
@@ -716,31 +758,34 @@ func (l rateLimit) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// potential pileup on the server.
|
||||
if err := l.Wait(ctx); err != nil {
|
||||
// Send an S3 compatible error, SlowDown.
|
||||
writeErrorResponse(w, ErrSlowDown, r.URL)
|
||||
writeErrorResponse(w, ErrSlowDown, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
l.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// requestIDHeaderHandler sets x-amz-request-id header.
|
||||
// Previously, this value was set right before a response
|
||||
// was sent to the client.So, logger and Error response XML
|
||||
// were not using this value.
|
||||
// This is set here so that this header can be logged as
|
||||
// part of the log entry and Error response XML.
|
||||
type requestIDHeaderHandler struct {
|
||||
// customHeaderHandler sets x-amz-request-id, x-minio-deployment-id header.
|
||||
// Previously, this value was set right before a response was sent to
|
||||
// the client. So, logger and Error response XML were not using this
|
||||
// value. This is set here so that this header can be logged as
|
||||
// part of the log entry, Error response XML and auditing.
|
||||
type customHeaderHandler struct {
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func addrequestIDHeader(h http.Handler) http.Handler {
|
||||
return requestIDHeaderHandler{handler: h}
|
||||
func addCustomHeaders(h http.Handler) http.Handler {
|
||||
return customHeaderHandler{handler: h}
|
||||
}
|
||||
|
||||
func (s requestIDHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Set unique request ID for each response.
|
||||
func (s customHeaderHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Set custom headers such as x-amz-request-id and x-minio-deployment-id
|
||||
// for each request.
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
|
||||
s.handler.ServeHTTP(w, r)
|
||||
if globalDeploymentID != "" {
|
||||
w.Header().Set(responseDeploymentIDKey, globalDeploymentID)
|
||||
}
|
||||
s.handler.ServeHTTP(logger.NewResponseWriter(w), r)
|
||||
}
|
||||
|
||||
type securityHeaderHandler struct {
|
||||
@@ -767,7 +812,7 @@ type criticalErrorHandler struct{ handler http.Handler }
|
||||
func (h criticalErrorHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
defer func() {
|
||||
if err := recover(); err == logger.ErrCritical { // handle
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
writeErrorResponse(w, ErrInternalError, r.URL, guessIsBrowserReq(r))
|
||||
} else if err != nil {
|
||||
panic(err) // forward other panic calls
|
||||
}
|
||||
@@ -786,7 +831,7 @@ func (h sseTLSHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodHead {
|
||||
writeErrorResponseHeadersOnly(w, ErrInsecureSSECustomerRequest)
|
||||
} else {
|
||||
writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL)
|
||||
writeErrorResponse(w, ErrInsecureSSECustomerRequest, r.URL, guessIsBrowserReq(r))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -78,9 +78,16 @@ func TestGuessIsRPC(t *testing.T) {
|
||||
if guessIsRPCReq(nil) {
|
||||
t.Fatal("Unexpected return for nil request")
|
||||
}
|
||||
|
||||
u, err := url.Parse("http://localhost:9000/minio/lock")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
r := &http.Request{
|
||||
Proto: "HTTP/1.0",
|
||||
Method: http.MethodPost,
|
||||
URL: u,
|
||||
}
|
||||
if !guessIsRPCReq(r) {
|
||||
t.Fatal("Test shouldn't fail for a possible net/rpc request.")
|
||||
|
||||
+17
-3
@@ -42,6 +42,8 @@ import (
|
||||
const (
|
||||
globalMinioCertExpireWarnDays = time.Hour * 24 * 30 // 30 days.
|
||||
|
||||
globalMinioDefaultPort = "9000"
|
||||
|
||||
globalMinioDefaultRegion = ""
|
||||
// This is a sha256 output of ``arn:aws:iam::minio:user/admin``,
|
||||
// this is kept in present form to be compatible with S3 owner ID
|
||||
@@ -90,6 +92,12 @@ const (
|
||||
maxLocationConstraintSize = 3 * humanize.MiByte
|
||||
)
|
||||
|
||||
var globalCLIContext = struct {
|
||||
JSON, Quiet bool
|
||||
Anonymous bool
|
||||
Addr string
|
||||
}{}
|
||||
|
||||
var (
|
||||
// Indicates the total number of erasure coded sets configured.
|
||||
globalXLSetCount int
|
||||
@@ -127,7 +135,7 @@ var (
|
||||
// Minio local server address (in `host:port` format)
|
||||
globalMinioAddr = ""
|
||||
// Minio default port, can be changed through command line.
|
||||
globalMinioPort = "9000"
|
||||
globalMinioPort = globalMinioDefaultPort
|
||||
// Holds the host that was passed using --address
|
||||
globalMinioHost = ""
|
||||
|
||||
@@ -230,8 +238,11 @@ var (
|
||||
globalKMSKeyID string
|
||||
// Allocated KMS
|
||||
globalKMS crypto.KMS
|
||||
// KMS config
|
||||
globalKMSConfig crypto.KMSConfig
|
||||
|
||||
// Auto-Encryption, if enabled, turns any non-SSE-C request
|
||||
// into an SSE-S3 request. If enabled a valid, non-empty KMS
|
||||
// configuration must be present.
|
||||
globalAutoEncryption bool
|
||||
|
||||
// Is compression include extensions/content-types set.
|
||||
globalIsEnvCompression bool
|
||||
@@ -255,6 +266,9 @@ var (
|
||||
// OPA policy system.
|
||||
globalPolicyOPA *iampolicy.Opa
|
||||
|
||||
// Deployment ID - unique per deployment
|
||||
globalDeploymentID string
|
||||
|
||||
// Add new variable global values here.
|
||||
)
|
||||
|
||||
|
||||
+29
-8
@@ -28,6 +28,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
httptracer "github.com/minio/minio/pkg/handlers"
|
||||
)
|
||||
@@ -133,6 +134,11 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set content-type to default value if it is not set.
|
||||
if _, ok := metadata["content-type"]; !ok {
|
||||
metadata["content-type"] = "application/octet-stream"
|
||||
}
|
||||
|
||||
// Success.
|
||||
return metadata, nil
|
||||
}
|
||||
@@ -176,13 +182,20 @@ func getRedirectPostRawQuery(objInfo ObjectInfo) string {
|
||||
return redirectValues.Encode()
|
||||
}
|
||||
|
||||
// Returns access key in the request Authorization header.
|
||||
func getReqAccessKey(r *http.Request, region string) (accessKey string) {
|
||||
cred, _, _ := getReqAccessKeyV4(r, region)
|
||||
// Returns access credentials in the request Authorization header.
|
||||
func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) {
|
||||
cred, _, _ = getReqAccessKeyV4(r, region)
|
||||
if cred.AccessKey == "" {
|
||||
cred, _, _ = getReqAccessKeyV2(r)
|
||||
}
|
||||
return cred.AccessKey
|
||||
if cred.AccessKey == "" {
|
||||
claims, owner, _ := webRequestAuthenticate(r)
|
||||
if owner {
|
||||
return globalServerConfig.GetCredential()
|
||||
}
|
||||
cred, _ = globalIAMSys.GetUser(claims.Subject)
|
||||
}
|
||||
return cred
|
||||
}
|
||||
|
||||
// Extract request params to be sent with event notifiation.
|
||||
@@ -192,10 +205,12 @@ func extractReqParams(r *http.Request) map[string]string {
|
||||
}
|
||||
|
||||
region := globalServerConfig.GetRegion()
|
||||
cred := getReqAccessCred(r, region)
|
||||
|
||||
// Success.
|
||||
return map[string]string{
|
||||
"region": region,
|
||||
"accessKey": getReqAccessKey(r, region),
|
||||
"accessKey": cred.AccessKey,
|
||||
"sourceIPAddress": handlers.GetSourceIP(r),
|
||||
// Add more fields here.
|
||||
}
|
||||
@@ -349,8 +364,14 @@ func getResource(path string, host string, domain string) (string, error) {
|
||||
return slashSeparator + pathJoin(bucket, path), nil
|
||||
}
|
||||
|
||||
// If none of the http routes match respond with MethodNotAllowed
|
||||
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL)
|
||||
// If none of the http routes match respond with MethodNotAllowed, in JSON
|
||||
func notFoundHandlerJSON(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// If none of the http routes match respond with MethodNotAllowed
|
||||
func notFoundHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponse(w, ErrMethodNotAllowed, r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// +build windows plan9
|
||||
// +build windows plan9 solaris
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
|
||||
@@ -178,6 +178,9 @@ type httpListener struct {
|
||||
// isRoutineNetErr returns true if error is due to a network timeout,
|
||||
// connect reset or io.EOF and false otherwise
|
||||
func isRoutineNetErr(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
if nErr, ok := err.(*net.OpError); ok {
|
||||
// Check if the error is a tcp connection reset
|
||||
if syscallErr, ok := nErr.Err.(*os.SyscallError); ok {
|
||||
@@ -188,7 +191,8 @@ func isRoutineNetErr(err error) bool {
|
||||
// Check if the error is a timeout
|
||||
return nErr.Timeout()
|
||||
}
|
||||
return err == io.EOF
|
||||
// check for io.EOF and also some times io.EOF is wrapped is another error type.
|
||||
return err == io.EOF || err.Error() == "EOF"
|
||||
}
|
||||
|
||||
// start - starts separate goroutine for each TCP listener. A valid insecure/TLS HTTP new connection is passed to httpListener.acceptCh.
|
||||
@@ -395,8 +399,8 @@ func newHTTPListener(serverAddrs []string,
|
||||
|
||||
for _, serverAddr := range serverAddrs {
|
||||
var l net.Listener
|
||||
if l, err = listen("tcp4", serverAddr); err != nil {
|
||||
if l, err = fallbackListen("tcp4", serverAddr); err != nil {
|
||||
if l, err = listen("tcp", serverAddr); err != nil {
|
||||
if l, err = fallbackListen("tcp", serverAddr); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
@@ -801,6 +802,10 @@ func TestIgnoreErr(t *testing.T) {
|
||||
err: &net.OpError{Err: &myTimeoutErr{timeout: true}},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
err: errors.New("EOF"),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
err: &net.OpError{Err: &myTimeoutErr{timeout: false}},
|
||||
want: false,
|
||||
@@ -809,6 +814,10 @@ func TestIgnoreErr(t *testing.T) {
|
||||
err: io.ErrUnexpectedEOF,
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
err: nil,
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, tc := range testCases {
|
||||
|
||||
+1
-1
@@ -36,7 +36,7 @@ const (
|
||||
DefaultShutdownTimeout = 5 * time.Second
|
||||
|
||||
// DefaultTCPKeepAliveTimeout - default TCP keep alive timeout for accepted connection.
|
||||
DefaultTCPKeepAliveTimeout = 10 * time.Second
|
||||
DefaultTCPKeepAliveTimeout = 30 * time.Second
|
||||
|
||||
// DefaultReadTimeout - default timout to read data from accepted connection.
|
||||
DefaultReadTimeout = 5 * time.Minute
|
||||
|
||||
+2
-2
@@ -60,9 +60,9 @@ type IAMSys struct {
|
||||
iamCannedPolicyMap map[string]iampolicy.Policy
|
||||
}
|
||||
|
||||
// Load - load iam.json
|
||||
// Load - loads iam subsystem
|
||||
func (sys *IAMSys) Load(objAPI ObjectLayer) error {
|
||||
return sys.Init(objAPI)
|
||||
return sys.refresh(objAPI)
|
||||
}
|
||||
|
||||
// Init - initializes config system from iam.json
|
||||
|
||||
@@ -18,9 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"io/ioutil"
|
||||
@@ -76,15 +74,6 @@ func (lc localAdminClient) ServerInfo() (sid ServerInfoData, e error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// GetConfig - returns config.json of the local server.
|
||||
func (lc localAdminClient) GetConfig() ([]byte, error) {
|
||||
if globalServerConfig == nil {
|
||||
return nil, fmt.Errorf("config not present")
|
||||
}
|
||||
|
||||
return json.Marshal(globalServerConfig)
|
||||
}
|
||||
|
||||
// StartProfiling - starts profiling on the local server.
|
||||
func (lc localAdminClient) StartProfiling(profiler string) error {
|
||||
if globalProfiler != nil {
|
||||
|
||||
@@ -24,14 +24,6 @@ func TestLocalAdminClientSignalService(t *testing.T) {
|
||||
testAdminCmdRunnerSignalService(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
func TestLocalAdminClientReInitFormat(t *testing.T) {
|
||||
testAdminCmdRunnerReInitFormat(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
func TestLocalAdminClientServerInfo(t *testing.T) {
|
||||
testAdminCmdRunnerServerInfo(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
func TestLocalAdminClientGetConfig(t *testing.T) {
|
||||
testAdminCmdRunnerGetConfig(t, &localAdminClient{})
|
||||
}
|
||||
|
||||
@@ -1,97 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Represents the current version of audit log structure.
|
||||
const auditLogVersion = "1"
|
||||
|
||||
// AuditEntry - audit entry logs.
|
||||
type AuditEntry struct {
|
||||
Version string `json:"version"`
|
||||
DeploymentID string `json:"deploymentid,omitempty"`
|
||||
Time string `json:"time"`
|
||||
API *api `json:"api,omitempty"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
ReqQuery map[string]string `json:"requestQuery,omitempty"`
|
||||
ReqHeader map[string]string `json:"requestHeader,omitempty"`
|
||||
RespHeader map[string]string `json:"responseHeader,omitempty"`
|
||||
}
|
||||
|
||||
// AuditTargets is the list of enabled audit loggers
|
||||
var AuditTargets = []LoggingTarget{}
|
||||
|
||||
// AddAuditTarget adds a new audit logger target to the
|
||||
// list of enabled loggers
|
||||
func AddAuditTarget(t LoggingTarget) {
|
||||
AuditTargets = append(AuditTargets, t)
|
||||
}
|
||||
|
||||
// AuditLog - logs audit logs to all targets.
|
||||
func AuditLog(ctx context.Context, w http.ResponseWriter, r *http.Request) {
|
||||
if Disable {
|
||||
return
|
||||
}
|
||||
|
||||
req := GetReqInfo(ctx)
|
||||
if req == nil {
|
||||
req = &ReqInfo{API: "SYSTEM"}
|
||||
}
|
||||
|
||||
reqQuery := make(map[string]string)
|
||||
for k, v := range r.URL.Query() {
|
||||
reqQuery[k] = strings.Join(v, ",")
|
||||
}
|
||||
reqHeader := make(map[string]string)
|
||||
for k, v := range r.Header {
|
||||
reqHeader[k] = strings.Join(v, ",")
|
||||
}
|
||||
respHeader := make(map[string]string)
|
||||
for k, v := range w.Header() {
|
||||
respHeader[k] = strings.Join(v, ",")
|
||||
}
|
||||
|
||||
// Send audit logs only to http targets.
|
||||
for _, t := range AuditTargets {
|
||||
t.send(AuditEntry{
|
||||
Version: auditLogVersion,
|
||||
DeploymentID: deploymentID,
|
||||
RemoteHost: req.RemoteHost,
|
||||
RequestID: req.RequestID,
|
||||
UserAgent: req.UserAgent,
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
API: &api{
|
||||
Name: req.API,
|
||||
Args: &args{
|
||||
Bucket: req.BucketName,
|
||||
Object: req.ObjectName,
|
||||
},
|
||||
},
|
||||
ReqQuery: reqQuery,
|
||||
ReqHeader: reqHeader,
|
||||
RespHeader: respHeader,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/cmd/logger/message/audit"
|
||||
)
|
||||
|
||||
// ResponseWriter - is a wrapper to trap the http response status code.
|
||||
type ResponseWriter struct {
|
||||
http.ResponseWriter
|
||||
statusCode int
|
||||
}
|
||||
|
||||
// NewResponseWriter - returns a wrapped response writer to trap
|
||||
// http status codes for auditiing purposes.
|
||||
func NewResponseWriter(w http.ResponseWriter) *ResponseWriter {
|
||||
return &ResponseWriter{w, http.StatusOK}
|
||||
}
|
||||
|
||||
// WriteHeader - writes http status code
|
||||
func (lrw *ResponseWriter) WriteHeader(code int) {
|
||||
lrw.statusCode = code
|
||||
lrw.ResponseWriter.WriteHeader(code)
|
||||
}
|
||||
|
||||
// Flush - Calls the underlying Flush.
|
||||
func (lrw *ResponseWriter) Flush() {
|
||||
lrw.ResponseWriter.(http.Flusher).Flush()
|
||||
}
|
||||
|
||||
// AuditTargets is the list of enabled audit loggers
|
||||
var AuditTargets = []Target{}
|
||||
|
||||
// AddAuditTarget adds a new audit logger target to the
|
||||
// list of enabled loggers
|
||||
func AddAuditTarget(t Target) {
|
||||
AuditTargets = append(AuditTargets, t)
|
||||
}
|
||||
|
||||
// AuditLog - logs audit logs to all audit targets.
|
||||
func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}) {
|
||||
var statusCode int
|
||||
lrw, ok := w.(*ResponseWriter)
|
||||
if ok {
|
||||
statusCode = lrw.statusCode
|
||||
}
|
||||
// Send audit logs only to http targets.
|
||||
for _, t := range AuditTargets {
|
||||
_ = t.Send(audit.ToEntry(w, r, api, statusCode, reqClaims))
|
||||
}
|
||||
}
|
||||
+150
-69
@@ -19,87 +19,168 @@ package logger
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
c "github.com/minio/mc/pkg/console"
|
||||
"github.com/minio/minio/cmd/logger/message/log"
|
||||
)
|
||||
|
||||
// ConsoleTarget implements loggerTarget to send log
|
||||
// in plain or json format to the standard output.
|
||||
type ConsoleTarget struct{}
|
||||
// Console interface describes the methods that need to be implemented to satisfy the interface requirements.
|
||||
type Console interface {
|
||||
json(msg string, args ...interface{})
|
||||
quiet(msg string, args ...interface{})
|
||||
pretty(msg string, args ...interface{})
|
||||
}
|
||||
|
||||
func (c *ConsoleTarget) send(e interface{}) error {
|
||||
entry, ok := e.(logEntry)
|
||||
if !ok {
|
||||
return fmt.Errorf("Uexpected log entry structure %#v", e)
|
||||
func consoleLog(console Console, msg string, args ...interface{}) {
|
||||
switch {
|
||||
case jsonFlag:
|
||||
// Strip escape control characters from json message
|
||||
msg = ansiRE.ReplaceAllLiteralString(msg, "")
|
||||
console.json(msg, args...)
|
||||
case quietFlag:
|
||||
console.quiet(msg, args...)
|
||||
default:
|
||||
console.pretty(msg, args...)
|
||||
}
|
||||
if jsonFlag {
|
||||
logJSON, err := json.Marshal(&entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Fatal prints only fatal error message with no stack trace
|
||||
// it will be called for input validation failures
|
||||
func Fatal(err error, msg string, data ...interface{}) {
|
||||
fatal(err, msg, data...)
|
||||
}
|
||||
|
||||
func fatal(err error, msg string, data ...interface{}) {
|
||||
var errMsg string
|
||||
if msg != "" {
|
||||
errMsg = errorFmtFunc(fmt.Sprintf(msg, data...), err, jsonFlag)
|
||||
} else {
|
||||
errMsg = err.Error()
|
||||
}
|
||||
consoleLog(fatalMessage, errMsg)
|
||||
}
|
||||
|
||||
var fatalMessage fatalMsg
|
||||
|
||||
type fatalMsg struct {
|
||||
}
|
||||
|
||||
func (f fatalMsg) json(msg string, args ...interface{}) {
|
||||
logJSON, err := json.Marshal(&log.Entry{
|
||||
Level: FatalLvl.String(),
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Trace: &log.Trace{Message: fmt.Sprintf(msg, args...), Source: []string{getSource(6)}},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(string(logJSON))
|
||||
|
||||
os.Exit(1)
|
||||
|
||||
}
|
||||
|
||||
func (f fatalMsg) quiet(msg string, args ...interface{}) {
|
||||
f.pretty(msg, args...)
|
||||
}
|
||||
|
||||
var (
|
||||
logTag = "ERROR"
|
||||
logBanner = ColorBgRed(ColorFgWhite(ColorBold(logTag))) + " "
|
||||
emptyBanner = ColorBgRed(strings.Repeat(" ", len(logTag))) + " "
|
||||
bannerWidth = len(logTag) + 1
|
||||
)
|
||||
|
||||
func (f fatalMsg) pretty(msg string, args ...interface{}) {
|
||||
// Build the passed error message
|
||||
errMsg := fmt.Sprintf(msg, args...)
|
||||
|
||||
tagPrinted := false
|
||||
|
||||
// Print the error message: the following code takes care
|
||||
// of splitting error text and always pretty printing the
|
||||
// red banner along with the error message. Since the error
|
||||
// message itself contains some colored text, we needed
|
||||
// to use some ANSI control escapes to cursor color state
|
||||
// and freely move in the screen.
|
||||
for _, line := range strings.Split(errMsg, "\n") {
|
||||
if len(line) == 0 {
|
||||
// No more text to print, just quit.
|
||||
break
|
||||
}
|
||||
fmt.Println(string(logJSON))
|
||||
return nil
|
||||
}
|
||||
|
||||
trace := make([]string, len(entry.Trace.Source))
|
||||
|
||||
// Add a sequence number and formatting for each stack trace
|
||||
// No formatting is required for the first entry
|
||||
for i, element := range entry.Trace.Source {
|
||||
trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
|
||||
}
|
||||
|
||||
tagString := ""
|
||||
for key, value := range entry.Trace.Variables {
|
||||
if value != "" {
|
||||
if tagString != "" {
|
||||
tagString += ", "
|
||||
for {
|
||||
// Save the attributes of the current cursor helps
|
||||
// us save the text color of the passed error message
|
||||
ansiSaveAttributes()
|
||||
// Print banner with or without the log tag
|
||||
if !tagPrinted {
|
||||
fmt.Print(logBanner)
|
||||
tagPrinted = true
|
||||
} else {
|
||||
fmt.Print(emptyBanner)
|
||||
}
|
||||
tagString += key + "=" + value
|
||||
// Restore the text color of the error message
|
||||
ansiRestoreAttributes()
|
||||
ansiMoveRight(bannerWidth)
|
||||
// Continue error message printing
|
||||
fmt.Println(line)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
apiString := "API: " + entry.API.Name + "("
|
||||
if entry.API.Args != nil && entry.API.Args.Bucket != "" {
|
||||
apiString = apiString + "bucket=" + entry.API.Args.Bucket
|
||||
}
|
||||
if entry.API.Args != nil && entry.API.Args.Object != "" {
|
||||
apiString = apiString + ", object=" + entry.API.Args.Object
|
||||
}
|
||||
apiString += ")"
|
||||
timeString := "Time: " + time.Now().Format(loggerTimeFormat)
|
||||
|
||||
var requestID string
|
||||
if entry.RequestID != "" {
|
||||
requestID = "\nRequestID: " + entry.RequestID
|
||||
}
|
||||
|
||||
var remoteHost string
|
||||
if entry.RemoteHost != "" {
|
||||
remoteHost = "\nRemoteHost: " + entry.RemoteHost
|
||||
}
|
||||
|
||||
var userAgent string
|
||||
if entry.UserAgent != "" {
|
||||
userAgent = "\nUserAgent: " + entry.UserAgent
|
||||
}
|
||||
|
||||
if len(entry.Trace.Variables) > 0 {
|
||||
tagString = "\n " + tagString
|
||||
}
|
||||
|
||||
var msg = colorFgRed(colorBold(entry.Trace.Message))
|
||||
var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
|
||||
apiString, timeString, requestID, remoteHost, userAgent,
|
||||
msg, tagString, strings.Join(trace, "\n"))
|
||||
|
||||
fmt.Println(output)
|
||||
return nil
|
||||
// Exit because this is a fatal error message
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// NewConsole initializes a new logger target
|
||||
// which prints log directly in the standard
|
||||
// output.
|
||||
func NewConsole() LoggingTarget {
|
||||
return &ConsoleTarget{}
|
||||
type infoMsg struct{}
|
||||
|
||||
var info infoMsg
|
||||
|
||||
func (i infoMsg) json(msg string, args ...interface{}) {
|
||||
logJSON, err := json.Marshal(&log.Entry{
|
||||
Level: InformationLvl.String(),
|
||||
Message: fmt.Sprintf(msg, args...),
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(string(logJSON))
|
||||
}
|
||||
|
||||
func (i infoMsg) quiet(msg string, args ...interface{}) {
|
||||
i.pretty(msg, args...)
|
||||
}
|
||||
|
||||
func (i infoMsg) pretty(msg string, args ...interface{}) {
|
||||
c.Printf(msg, args...)
|
||||
}
|
||||
|
||||
// Info :
|
||||
func Info(msg string, data ...interface{}) {
|
||||
consoleLog(info, msg+"\n", data...)
|
||||
}
|
||||
|
||||
var startupMessage startUpMsg
|
||||
|
||||
type startUpMsg struct {
|
||||
}
|
||||
|
||||
func (s startUpMsg) json(msg string, args ...interface{}) {
|
||||
}
|
||||
|
||||
func (s startUpMsg) quiet(msg string, args ...interface{}) {
|
||||
}
|
||||
|
||||
func (s startUpMsg) pretty(msg string, args ...interface{}) {
|
||||
c.Printf(msg, args...)
|
||||
}
|
||||
|
||||
// StartupMessage :
|
||||
func StartupMessage(msg string, data ...interface{}) {
|
||||
consoleLog(startupMessage, msg+"\n", data...)
|
||||
}
|
||||
|
||||
+81
-209
@@ -18,23 +18,31 @@ package logger
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"go/build"
|
||||
"os"
|
||||
"hash"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
c "github.com/minio/mc/pkg/console"
|
||||
"github.com/minio/highwayhash"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/cmd/logger/message/log"
|
||||
)
|
||||
|
||||
var (
|
||||
// HighwayHash key for logging in anonymous mode
|
||||
magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0")
|
||||
// HighwayHash hasher for logging in anonymous mode
|
||||
loggerHighwayHasher hash.Hash
|
||||
)
|
||||
|
||||
// Disable disables all logging, false by default. (used for "go test")
|
||||
var Disable = false
|
||||
|
||||
var trimStrings []string
|
||||
|
||||
// Level type
|
||||
type Level int8
|
||||
|
||||
@@ -45,7 +53,10 @@ const (
|
||||
FatalLvl
|
||||
)
|
||||
|
||||
const loggerTimeFormat string = "15:04:05 MST 01/02/2006"
|
||||
var trimStrings []string
|
||||
|
||||
// TimeFormat - logging time format.
|
||||
const TimeFormat string = "15:04:05 MST 01/02/2006"
|
||||
|
||||
// List of error strings to be ignored by LogIf
|
||||
const (
|
||||
@@ -91,79 +102,39 @@ func (level Level) String() string {
|
||||
return lvlStr
|
||||
}
|
||||
|
||||
// Console interface describes the methods that needs to be implemented to satisfy the interface requirements.
|
||||
type Console interface {
|
||||
json(msg string, args ...interface{})
|
||||
quiet(msg string, args ...interface{})
|
||||
pretty(msg string, args ...interface{})
|
||||
}
|
||||
|
||||
func consoleLog(console Console, msg string, args ...interface{}) {
|
||||
switch {
|
||||
case jsonFlag:
|
||||
// Strip escape control characters from json message
|
||||
msg = ansiRE.ReplaceAllLiteralString(msg, "")
|
||||
console.json(msg, args...)
|
||||
case quiet:
|
||||
console.quiet(msg, args...)
|
||||
default:
|
||||
console.pretty(msg, args...)
|
||||
}
|
||||
}
|
||||
|
||||
type traceEntry struct {
|
||||
Message string `json:"message,omitempty"`
|
||||
Source []string `json:"source,omitempty"`
|
||||
Variables map[string]string `json:"variables,omitempty"`
|
||||
}
|
||||
|
||||
type args struct {
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
Object string `json:"object,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
type api struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Args *args `json:"args,omitempty"`
|
||||
}
|
||||
|
||||
type logEntry struct {
|
||||
DeploymentID string `json:"deploymentid,omitempty"`
|
||||
Level string `json:"level"`
|
||||
Time string `json:"time"`
|
||||
API *api `json:"api,omitempty"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Trace *traceEntry `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// quiet: Hide startup messages if enabled
|
||||
// quietFlag: Hide startup messages if enabled
|
||||
// jsonFlag: Display in JSON format, if enabled
|
||||
var (
|
||||
quiet, jsonFlag bool
|
||||
quietFlag, jsonFlag, anonFlag bool
|
||||
// Custom function to format error
|
||||
errorFmtFunc func(string, error, bool) string
|
||||
|
||||
deploymentID string
|
||||
)
|
||||
|
||||
// SetDeploymentID - Used to set the deployment ID, in XL and FS mode
|
||||
func SetDeploymentID(id string) {
|
||||
deploymentID = id
|
||||
}
|
||||
|
||||
// EnableQuiet - turns quiet option on.
|
||||
func EnableQuiet() {
|
||||
quiet = true
|
||||
quietFlag = true
|
||||
}
|
||||
|
||||
// EnableJSON - outputs logs in json format.
|
||||
func EnableJSON() {
|
||||
jsonFlag = true
|
||||
quiet = true
|
||||
quietFlag = true
|
||||
}
|
||||
|
||||
// EnableAnonymous - turns anonymous flag
|
||||
// to avoid printing sensitive information.
|
||||
func EnableAnonymous() {
|
||||
anonFlag = true
|
||||
}
|
||||
|
||||
// IsJSON - returns true if jsonFlag is true
|
||||
func IsJSON() bool {
|
||||
return jsonFlag
|
||||
}
|
||||
|
||||
// IsQuiet - returns true if quietFlag is true
|
||||
func IsQuiet() bool {
|
||||
return quietFlag
|
||||
}
|
||||
|
||||
// RegisterUIError registers the specified rendering function. This latter
|
||||
@@ -172,6 +143,17 @@ func RegisterUIError(f func(string, error, bool) string) {
|
||||
errorFmtFunc = f
|
||||
}
|
||||
|
||||
// Remove any duplicates and return unique entries.
|
||||
func uniqueEntries(paths []string) []string {
|
||||
m := make(set.StringSet)
|
||||
for _, p := range paths {
|
||||
if !m.Contains(p) {
|
||||
m.Add(p)
|
||||
}
|
||||
}
|
||||
return m.ToSlice()
|
||||
}
|
||||
|
||||
// Init sets the trimStrings to possible GOPATHs
|
||||
// and GOROOT directories. Also append github.com/minio/minio
|
||||
// This is done to clean up the filename, when stack trace is
|
||||
@@ -222,6 +204,8 @@ func Init(goPath string, goRoot string) {
|
||||
// paths like "{GOROOT}/src/github.com/minio/minio"
|
||||
// and "{GOPATH}/src/github.com/minio/minio"
|
||||
trimStrings = append(trimStrings, filepath.Join("github.com", "minio", "minio")+string(filepath.Separator))
|
||||
|
||||
loggerHighwayHasher, _ = highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit
|
||||
}
|
||||
|
||||
func trimTrace(f string) string {
|
||||
@@ -274,6 +258,14 @@ func getTrace(traceLevel int) []string {
|
||||
return trace
|
||||
}
|
||||
|
||||
// Return the highway hash of the passed string
|
||||
func hashString(input string) string {
|
||||
defer loggerHighwayHasher.Reset()
|
||||
loggerHighwayHasher.Write([]byte(input))
|
||||
checksum := loggerHighwayHasher.Sum(nil)
|
||||
return hex.EncodeToString(checksum)
|
||||
}
|
||||
|
||||
// LogAlwaysIf prints a detailed error message during
|
||||
// the execution of the server.
|
||||
func LogAlwaysIf(ctx context.Context, err error) {
|
||||
@@ -326,20 +318,38 @@ func logIf(ctx context.Context, err error) {
|
||||
// Get the cause for the Error
|
||||
message := err.Error()
|
||||
|
||||
entry := logEntry{
|
||||
DeploymentID: deploymentID,
|
||||
entry := log.Entry{
|
||||
DeploymentID: req.DeploymentID,
|
||||
Level: ErrorLvl.String(),
|
||||
RemoteHost: req.RemoteHost,
|
||||
RequestID: req.RequestID,
|
||||
UserAgent: req.UserAgent,
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
API: &api{Name: API, Args: &args{Bucket: req.BucketName, Object: req.ObjectName}},
|
||||
Trace: &traceEntry{Message: message, Source: trace, Variables: tags},
|
||||
API: &log.API{
|
||||
Name: API,
|
||||
Args: &log.Args{
|
||||
Bucket: req.BucketName,
|
||||
Object: req.ObjectName,
|
||||
},
|
||||
},
|
||||
Trace: &log.Trace{
|
||||
Message: message,
|
||||
Source: trace,
|
||||
Variables: tags,
|
||||
},
|
||||
}
|
||||
|
||||
if anonFlag {
|
||||
entry.API.Args.Bucket = hashString(entry.API.Args.Bucket)
|
||||
entry.API.Args.Object = hashString(entry.API.Args.Object)
|
||||
entry.RemoteHost = hashString(entry.RemoteHost)
|
||||
entry.Message = reflect.TypeOf(err).String()
|
||||
entry.Trace.Variables = make(map[string]string)
|
||||
}
|
||||
|
||||
// Iterate over all logger targets to send the log entry
|
||||
for _, t := range Targets {
|
||||
t.send(entry)
|
||||
t.Send(entry)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -362,141 +372,3 @@ func FatalIf(err error, msg string, data ...interface{}) {
|
||||
}
|
||||
fatal(err, msg, data...)
|
||||
}
|
||||
|
||||
// Fatal prints only fatal error message without no stack trace
|
||||
// it will be called for input validation failures
|
||||
func Fatal(err error, msg string, data ...interface{}) {
|
||||
fatal(err, msg, data...)
|
||||
}
|
||||
|
||||
func fatal(err error, msg string, data ...interface{}) {
|
||||
var errMsg string
|
||||
if msg != "" {
|
||||
errMsg = errorFmtFunc(fmt.Sprintf(msg, data...), err, jsonFlag)
|
||||
} else {
|
||||
errMsg = err.Error()
|
||||
}
|
||||
consoleLog(fatalMessage, errMsg)
|
||||
}
|
||||
|
||||
var fatalMessage fatalMsg
|
||||
|
||||
type fatalMsg struct {
|
||||
}
|
||||
|
||||
func (f fatalMsg) json(msg string, args ...interface{}) {
|
||||
logJSON, err := json.Marshal(&logEntry{
|
||||
Level: FatalLvl.String(),
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
Trace: &traceEntry{Message: fmt.Sprintf(msg, args...), Source: []string{getSource(6)}},
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(string(logJSON))
|
||||
|
||||
os.Exit(1)
|
||||
|
||||
}
|
||||
|
||||
func (f fatalMsg) quiet(msg string, args ...interface{}) {
|
||||
f.pretty(msg, args...)
|
||||
}
|
||||
|
||||
var (
|
||||
logTag = "ERROR"
|
||||
logBanner = colorBgRed(colorFgWhite(colorBold(logTag))) + " "
|
||||
emptyBanner = colorBgRed(strings.Repeat(" ", len(logTag))) + " "
|
||||
bannerWidth = len(logTag) + 1
|
||||
)
|
||||
|
||||
func (f fatalMsg) pretty(msg string, args ...interface{}) {
|
||||
// Build the passed error message
|
||||
errMsg := fmt.Sprintf(msg, args...)
|
||||
|
||||
tagPrinted := false
|
||||
|
||||
// Print the error message: the following code takes care
|
||||
// of splitting error text and always pretty printing the
|
||||
// red banner along with the error message. Since the error
|
||||
// message itself contains some colored text, we needed
|
||||
// to use some ANSI control escapes to cursor color state
|
||||
// and freely move in the screen.
|
||||
for _, line := range strings.Split(errMsg, "\n") {
|
||||
if len(line) == 0 {
|
||||
// No more text to print, just quit.
|
||||
break
|
||||
}
|
||||
|
||||
for {
|
||||
// Save the attributes of the current cursor helps
|
||||
// us save the text color of the passed error message
|
||||
ansiSaveAttributes()
|
||||
// Print banner with or without the log tag
|
||||
if !tagPrinted {
|
||||
fmt.Print(logBanner)
|
||||
tagPrinted = true
|
||||
} else {
|
||||
fmt.Print(emptyBanner)
|
||||
}
|
||||
// Restore the text color of the error message
|
||||
ansiRestoreAttributes()
|
||||
ansiMoveRight(bannerWidth)
|
||||
// Continue error message printing
|
||||
fmt.Println(line)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Exit because this is a fatal error message
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
type infoMsg struct{}
|
||||
|
||||
var info infoMsg
|
||||
|
||||
func (i infoMsg) json(msg string, args ...interface{}) {
|
||||
logJSON, err := json.Marshal(&logEntry{
|
||||
Level: InformationLvl.String(),
|
||||
Message: fmt.Sprintf(msg, args...),
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Println(string(logJSON))
|
||||
}
|
||||
|
||||
func (i infoMsg) quiet(msg string, args ...interface{}) {
|
||||
i.pretty(msg, args...)
|
||||
}
|
||||
|
||||
func (i infoMsg) pretty(msg string, args ...interface{}) {
|
||||
c.Printf(msg, args...)
|
||||
}
|
||||
|
||||
// Info :
|
||||
func Info(msg string, data ...interface{}) {
|
||||
consoleLog(info, msg+"\n", data...)
|
||||
}
|
||||
|
||||
var startupMessage startUpMsg
|
||||
|
||||
type startUpMsg struct {
|
||||
}
|
||||
|
||||
func (s startUpMsg) json(msg string, args ...interface{}) {
|
||||
}
|
||||
|
||||
func (s startUpMsg) quiet(msg string, args ...interface{}) {
|
||||
}
|
||||
|
||||
func (s startUpMsg) pretty(msg string, args ...interface{}) {
|
||||
c.Printf(msg, args...)
|
||||
}
|
||||
|
||||
// StartupMessage :
|
||||
func StartupMessage(msg string, data ...interface{}) {
|
||||
consoleLog(startupMessage, msg+"\n", data...)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package audit
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
)
|
||||
|
||||
// Version - represents the current version of audit log structure.
|
||||
const Version = "1"
|
||||
|
||||
// Entry - audit entry logs.
|
||||
type Entry struct {
|
||||
Version string `json:"version"`
|
||||
DeploymentID string `json:"deploymentid,omitempty"`
|
||||
Time string `json:"time"`
|
||||
API struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
Object string `json:"object,omitempty"`
|
||||
Status string `json:"status,omitempty"`
|
||||
StatusCode int `json:"statusCode,omitempty"`
|
||||
} `json:"api"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
ReqClaims map[string]interface{} `json:"requestClaims,omitempty"`
|
||||
ReqQuery map[string]string `json:"requestQuery,omitempty"`
|
||||
ReqHeader map[string]string `json:"requestHeader,omitempty"`
|
||||
RespHeader map[string]string `json:"responseHeader,omitempty"`
|
||||
}
|
||||
|
||||
// ToEntry - constructs an audit entry object.
|
||||
func ToEntry(w http.ResponseWriter, r *http.Request, api string, statusCode int, reqClaims map[string]interface{}) Entry {
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
object := vars["object"]
|
||||
|
||||
reqQuery := make(map[string]string)
|
||||
for k, v := range r.URL.Query() {
|
||||
reqQuery[k] = strings.Join(v, ",")
|
||||
}
|
||||
reqHeader := make(map[string]string)
|
||||
for k, v := range r.Header {
|
||||
reqHeader[k] = strings.Join(v, ",")
|
||||
}
|
||||
respHeader := make(map[string]string)
|
||||
for k, v := range w.Header() {
|
||||
respHeader[k] = strings.Join(v, ",")
|
||||
}
|
||||
respHeader["Etag"] = strings.Trim(respHeader["Etag"], `"`)
|
||||
|
||||
entry := Entry{
|
||||
Version: Version,
|
||||
DeploymentID: w.Header().Get("x-minio-deployment-id"),
|
||||
RemoteHost: handlers.GetSourceIP(r),
|
||||
RequestID: w.Header().Get("x-amz-request-id"),
|
||||
UserAgent: r.UserAgent(),
|
||||
Time: time.Now().UTC().Format(time.RFC3339Nano),
|
||||
ReqQuery: reqQuery,
|
||||
ReqHeader: reqHeader,
|
||||
ReqClaims: reqClaims,
|
||||
RespHeader: respHeader,
|
||||
}
|
||||
|
||||
entry.API.Name = api
|
||||
entry.API.Bucket = bucket
|
||||
entry.API.Object = object
|
||||
entry.API.Status = http.StatusText(statusCode)
|
||||
entry.API.StatusCode = statusCode
|
||||
|
||||
return entry
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package log
|
||||
|
||||
// Args - defines the arguments for the API.
|
||||
type Args struct {
|
||||
Bucket string `json:"bucket,omitempty"`
|
||||
Object string `json:"object,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
}
|
||||
|
||||
// Trace - defines the trace.
|
||||
type Trace struct {
|
||||
Message string `json:"message,omitempty"`
|
||||
Source []string `json:"source,omitempty"`
|
||||
Variables map[string]string `json:"variables,omitempty"`
|
||||
}
|
||||
|
||||
// API - defines the api type and its args.
|
||||
type API struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Args *Args `json:"args,omitempty"`
|
||||
}
|
||||
|
||||
// Entry - defines fields and values of each log entry.
|
||||
type Entry struct {
|
||||
DeploymentID string `json:"deploymentid,omitempty"`
|
||||
Level string `json:"level"`
|
||||
Time string `json:"time"`
|
||||
API *API `json:"api,omitempty"`
|
||||
RemoteHost string `json:"remotehost,omitempty"`
|
||||
RequestID string `json:"requestID,omitempty"`
|
||||
UserAgent string `json:"userAgent,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Trace *Trace `json:"error,omitempty"`
|
||||
}
|
||||
+10
-8
@@ -35,22 +35,24 @@ type KeyVal struct {
|
||||
|
||||
// ReqInfo stores the request info.
|
||||
type ReqInfo struct {
|
||||
RemoteHost string // Client Host/IP
|
||||
UserAgent string // User Agent
|
||||
RequestID string // x-amz-request-id
|
||||
API string // API name - GetObject PutObject NewMultipartUpload etc.
|
||||
BucketName string // Bucket name
|
||||
ObjectName string // Object name
|
||||
tags []KeyVal // Any additional info not accommodated by above fields
|
||||
RemoteHost string // Client Host/IP
|
||||
UserAgent string // User Agent
|
||||
DeploymentID string // x-minio-deployment-id
|
||||
RequestID string // x-amz-request-id
|
||||
API string // API name - GetObject PutObject NewMultipartUpload etc.
|
||||
BucketName string // Bucket name
|
||||
ObjectName string // Object name
|
||||
tags []KeyVal // Any additional info not accommodated by above fields
|
||||
sync.RWMutex
|
||||
}
|
||||
|
||||
// NewReqInfo :
|
||||
func NewReqInfo(remoteHost, userAgent, requestID, api, bucket, object string) *ReqInfo {
|
||||
func NewReqInfo(remoteHost, userAgent, deploymentID, requestID, api, bucket, object string) *ReqInfo {
|
||||
req := ReqInfo{}
|
||||
req.RemoteHost = remoteHost
|
||||
req.UserAgent = userAgent
|
||||
req.API = api
|
||||
req.DeploymentID = deploymentID
|
||||
req.RequestID = requestID
|
||||
req.BucketName = bucket
|
||||
req.ObjectName = object
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package console
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/logger/message/log"
|
||||
)
|
||||
|
||||
// Target implements loggerTarget to send log
|
||||
// in plain or json format to the standard output.
|
||||
type Target struct{}
|
||||
|
||||
// Send log message 'e' to console
|
||||
func (c *Target) Send(e interface{}) error {
|
||||
entry, ok := e.(log.Entry)
|
||||
if !ok {
|
||||
return fmt.Errorf("Uexpected log entry structure %#v", e)
|
||||
}
|
||||
if logger.IsJSON() {
|
||||
logJSON, err := json.Marshal(&entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(string(logJSON))
|
||||
return nil
|
||||
}
|
||||
|
||||
trace := make([]string, len(entry.Trace.Source))
|
||||
|
||||
// Add a sequence number and formatting for each stack trace
|
||||
// No formatting is required for the first entry
|
||||
for i, element := range entry.Trace.Source {
|
||||
trace[i] = fmt.Sprintf("%8v: %s", i+1, element)
|
||||
}
|
||||
|
||||
tagString := ""
|
||||
for key, value := range entry.Trace.Variables {
|
||||
if value != "" {
|
||||
if tagString != "" {
|
||||
tagString += ", "
|
||||
}
|
||||
tagString += key + "=" + value
|
||||
}
|
||||
}
|
||||
|
||||
apiString := "API: " + entry.API.Name + "("
|
||||
if entry.API.Args != nil && entry.API.Args.Bucket != "" {
|
||||
apiString = apiString + "bucket=" + entry.API.Args.Bucket
|
||||
}
|
||||
if entry.API.Args != nil && entry.API.Args.Object != "" {
|
||||
apiString = apiString + ", object=" + entry.API.Args.Object
|
||||
}
|
||||
apiString += ")"
|
||||
timeString := "Time: " + time.Now().Format(logger.TimeFormat)
|
||||
|
||||
var requestID string
|
||||
if entry.RequestID != "" {
|
||||
requestID = "\nRequestID: " + entry.RequestID
|
||||
}
|
||||
|
||||
var remoteHost string
|
||||
if entry.RemoteHost != "" {
|
||||
remoteHost = "\nRemoteHost: " + entry.RemoteHost
|
||||
}
|
||||
|
||||
var userAgent string
|
||||
if entry.UserAgent != "" {
|
||||
userAgent = "\nUserAgent: " + entry.UserAgent
|
||||
}
|
||||
|
||||
if len(entry.Trace.Variables) > 0 {
|
||||
tagString = "\n " + tagString
|
||||
}
|
||||
|
||||
var msg = logger.ColorFgRed(logger.ColorBold(entry.Trace.Message))
|
||||
var output = fmt.Sprintf("\n%s\n%s%s%s%s\nError: %s%s\n%s",
|
||||
apiString, timeString, requestID, remoteHost, userAgent,
|
||||
msg, tagString, strings.Join(trace, "\n"))
|
||||
|
||||
fmt.Println(output)
|
||||
return nil
|
||||
}
|
||||
|
||||
// New initializes a new logger target
|
||||
// which prints log directly in the standard
|
||||
// output.
|
||||
func New() *Target {
|
||||
return &Target{}
|
||||
}
|
||||
@@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package logger
|
||||
package http
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@@ -22,24 +22,24 @@ import (
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
gohttp "net/http"
|
||||
)
|
||||
|
||||
// HTTPTarget implements loggerTarget and sends the json
|
||||
// Target implements logger.Target and sends the json
|
||||
// format of a log entry to the configured http endpoint.
|
||||
// An internal buffer of logs is maintained but when the
|
||||
// buffer is full, new logs are just ignored and an error
|
||||
// is returned to the caller.
|
||||
type HTTPTarget struct {
|
||||
type Target struct {
|
||||
// Channel of log entries
|
||||
logCh chan interface{}
|
||||
|
||||
// HTTP(s) endpoint
|
||||
endpoint string
|
||||
client http.Client
|
||||
client gohttp.Client
|
||||
}
|
||||
|
||||
func (h *HTTPTarget) startHTTPLogger() {
|
||||
func (h *Target) startHTTPLogger() {
|
||||
// Create a routine which sends json logs received
|
||||
// from an internal channel.
|
||||
go func() {
|
||||
@@ -49,7 +49,7 @@ func (h *HTTPTarget) startHTTPLogger() {
|
||||
continue
|
||||
}
|
||||
|
||||
req, err := http.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
|
||||
req, err := gohttp.NewRequest("POST", h.endpoint, bytes.NewBuffer(logJSON))
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
resp, err := h.client.Do(req)
|
||||
@@ -65,12 +65,12 @@ func (h *HTTPTarget) startHTTPLogger() {
|
||||
}()
|
||||
}
|
||||
|
||||
// NewHTTP initializes a new logger target which
|
||||
// New initializes a new logger target which
|
||||
// sends log over http to the specified endpoint
|
||||
func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
|
||||
h := HTTPTarget{
|
||||
func New(endpoint string, transport *gohttp.Transport) *Target {
|
||||
h := Target{
|
||||
endpoint: endpoint,
|
||||
client: http.Client{
|
||||
client: gohttp.Client{
|
||||
Transport: transport,
|
||||
},
|
||||
logCh: make(chan interface{}, 10000),
|
||||
@@ -80,7 +80,8 @@ func NewHTTP(endpoint string, transport *http.Transport) LoggingTarget {
|
||||
return &h
|
||||
}
|
||||
|
||||
func (h *HTTPTarget) send(entry interface{}) error {
|
||||
// Send log message 'e' to http target.
|
||||
func (h *Target) Send(entry interface{}) error {
|
||||
select {
|
||||
case h.logCh <- entry:
|
||||
default:
|
||||
@@ -16,18 +16,18 @@
|
||||
|
||||
package logger
|
||||
|
||||
// LoggingTarget is the entity that we will receive
|
||||
// a single log entry and send it to the log target
|
||||
// e.g. send the log to a http server
|
||||
type LoggingTarget interface {
|
||||
send(entry interface{}) error
|
||||
// Target is the entity that we will receive
|
||||
// a single log entry and Send it to the log target
|
||||
// e.g. Send the log to a http server
|
||||
type Target interface {
|
||||
Send(entry interface{}) error
|
||||
}
|
||||
|
||||
// Targets is the set of enabled loggers
|
||||
var Targets = []LoggingTarget{}
|
||||
var Targets = []Target{}
|
||||
|
||||
// AddTarget adds a new logger target to the
|
||||
// list of enabled loggers
|
||||
func AddTarget(t LoggingTarget) {
|
||||
func AddTarget(t Target) {
|
||||
Targets = append(Targets, t)
|
||||
}
|
||||
|
||||
+4
-16
@@ -33,25 +33,25 @@ var (
|
||||
return isatty.IsTerminal(os.Stdout.Fd()) && isatty.IsTerminal(os.Stderr.Fd())
|
||||
}
|
||||
|
||||
colorBold = func() func(a ...interface{}) string {
|
||||
ColorBold = func() func(a ...interface{}) string {
|
||||
if isTerminal() {
|
||||
return color.New(color.Bold).SprintFunc()
|
||||
}
|
||||
return fmt.Sprint
|
||||
}()
|
||||
colorFgRed = func() func(format string, a ...interface{}) string {
|
||||
ColorFgRed = func() func(format string, a ...interface{}) string {
|
||||
if isTerminal() {
|
||||
return color.New(color.FgRed).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
colorBgRed = func() func(format string, a ...interface{}) string {
|
||||
ColorBgRed = func() func(format string, a ...interface{}) string {
|
||||
if isTerminal() {
|
||||
return color.New(color.BgRed).SprintfFunc()
|
||||
}
|
||||
return fmt.Sprintf
|
||||
}()
|
||||
colorFgWhite = func() func(format string, a ...interface{}) string {
|
||||
ColorFgWhite = func() func(format string, a ...interface{}) string {
|
||||
if isTerminal() {
|
||||
return color.New(color.FgWhite).SprintfFunc()
|
||||
}
|
||||
@@ -83,17 +83,5 @@ func ansiRestoreAttributes() {
|
||||
if isTerminal() {
|
||||
ansiEscape("8")
|
||||
}
|
||||
}
|
||||
|
||||
func uniqueEntries(paths []string) []string {
|
||||
found := map[string]bool{}
|
||||
unqiue := []string{}
|
||||
|
||||
for v := range paths {
|
||||
if _, ok := found[paths[v]]; !ok {
|
||||
found[paths[v]] = true
|
||||
unqiue = append(unqiue, paths[v])
|
||||
}
|
||||
}
|
||||
return unqiue
|
||||
}
|
||||
|
||||
@@ -44,6 +44,10 @@ var globalFlags = []cli.Flag{
|
||||
Name: "quiet",
|
||||
Usage: "Disable startup information.",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "anonymous",
|
||||
Usage: "Hide sensitive information from logging.",
|
||||
},
|
||||
cli.BoolFlag{
|
||||
Name: "json",
|
||||
Usage: "Output server logs and startup information in json format.",
|
||||
|
||||
+57
-20
@@ -37,14 +37,21 @@ import (
|
||||
// IPv4 addresses of local host.
|
||||
var localIP4 = mustGetLocalIP4()
|
||||
|
||||
// IPv6 address of local host.
|
||||
var localIP6 = mustGetLocalIP6()
|
||||
|
||||
// mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal.
|
||||
func mustSplitHostPort(hostPort string) (host, port string) {
|
||||
host, port, err := net.SplitHostPort(hostPort)
|
||||
// Strip off IPv6 zone information.
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
logger.FatalIf(err, "Unable to split host port %s", hostPort)
|
||||
return host, port
|
||||
}
|
||||
|
||||
// mustGetLocalIP4 returns IPv4 addresses of local host. It panics on error.
|
||||
// mustGetLocalIP4 returns IPv4 addresses of localhost. It panics on error.
|
||||
func mustGetLocalIP4() (ipList set.StringSet) {
|
||||
ipList = set.NewStringSet()
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
@@ -67,8 +74,31 @@ func mustGetLocalIP4() (ipList set.StringSet) {
|
||||
return ipList
|
||||
}
|
||||
|
||||
// getHostIP4 returns IPv4 address of given host.
|
||||
func getHostIP4(host string) (ipList set.StringSet, err error) {
|
||||
// mustGetLocalIP6 returns IPv6 addresses of localhost. It panics on error.
|
||||
func mustGetLocalIP6() (ipList set.StringSet) {
|
||||
ipList = set.NewStringSet()
|
||||
addrs, err := net.InterfaceAddrs()
|
||||
logger.FatalIf(err, "Unable to get IP addresses of this host")
|
||||
|
||||
for _, addr := range addrs {
|
||||
var ip net.IP
|
||||
switch v := addr.(type) {
|
||||
case *net.IPNet:
|
||||
ip = v.IP
|
||||
case *net.IPAddr:
|
||||
ip = v.IP
|
||||
}
|
||||
|
||||
if ip.To4() == nil {
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
}
|
||||
|
||||
return ipList
|
||||
}
|
||||
|
||||
// getHostIP returns IP address of given host.
|
||||
func getHostIP(host string) (ipList set.StringSet, err error) {
|
||||
var ips []net.IP
|
||||
|
||||
if ips, err = net.LookupIP(host); err != nil {
|
||||
@@ -110,9 +140,7 @@ func getHostIP4(host string) (ipList set.StringSet, err error) {
|
||||
|
||||
ipList = set.NewStringSet()
|
||||
for _, ip := range ips {
|
||||
if ip.To4() != nil {
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
ipList.Add(ip.String())
|
||||
}
|
||||
|
||||
return ipList, err
|
||||
@@ -169,29 +197,32 @@ func sortIPs(ipList []string) []string {
|
||||
return append(nonIPs, ips...)
|
||||
}
|
||||
|
||||
func getAPIEndpoints(serverAddr string) (apiEndpoints []string) {
|
||||
host, port := mustSplitHostPort(serverAddr)
|
||||
|
||||
func getAPIEndpoints() (apiEndpoints []string) {
|
||||
var ipList []string
|
||||
if host == "" {
|
||||
if globalMinioHost == "" {
|
||||
ipList = sortIPs(localIP4.ToSlice())
|
||||
ipList = append(ipList, localIP6.ToSlice()...)
|
||||
} else {
|
||||
ipList = []string{host}
|
||||
ipList = []string{globalMinioHost}
|
||||
}
|
||||
|
||||
for _, ip := range ipList {
|
||||
apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), ip, port))
|
||||
apiEndpoints = append(apiEndpoints, fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(ip, globalMinioPort)))
|
||||
}
|
||||
|
||||
return apiEndpoints
|
||||
}
|
||||
|
||||
// isHostIPv4 - helper for validating if the provided arg is an ip address.
|
||||
func isHostIPv4(ipAddress string) bool {
|
||||
// isHostIP - helper for validating if the provided arg is an ip address.
|
||||
func isHostIP(ipAddress string) bool {
|
||||
host, _, err := net.SplitHostPort(ipAddress)
|
||||
if err != nil {
|
||||
host = ipAddress
|
||||
}
|
||||
// Strip off IPv6 zone information.
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
return net.ParseIP(host) != nil
|
||||
}
|
||||
|
||||
@@ -291,19 +322,20 @@ func extractHostPort(hostAddr string) (string, string, error) {
|
||||
// correspond to one of the local IP of the
|
||||
// current machine
|
||||
func isLocalHost(host string) (bool, error) {
|
||||
hostIPs, err := getHostIP4(host)
|
||||
hostIPs, err := getHostIP(host)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// If intersection of two IP sets is not empty, then the host is local host.
|
||||
isLocal := !localIP4.Intersection(hostIPs).IsEmpty()
|
||||
return isLocal, nil
|
||||
// If intersection of two IP sets is not empty, then the host is localhost.
|
||||
isLocalv4 := !localIP4.Intersection(hostIPs).IsEmpty()
|
||||
isLocalv6 := !localIP6.Intersection(hostIPs).IsEmpty()
|
||||
return isLocalv4 || isLocalv6, nil
|
||||
}
|
||||
|
||||
// sameLocalAddrs - returns true if two addresses, even with different
|
||||
// formats, point to the same machine, e.g:
|
||||
// ':9000' and 'http://localhost:9000/' will return true
|
||||
// ':9000' and 'http://localhost:9000/' will return true
|
||||
func sameLocalAddrs(addr1, addr2 string) (bool, error) {
|
||||
|
||||
// Extract host & port from given parameters
|
||||
@@ -355,6 +387,11 @@ func CheckLocalServerAddr(serverAddr string) error {
|
||||
return uiErrInvalidAddressFlag(err)
|
||||
}
|
||||
|
||||
// Strip off IPv6 zone information.
|
||||
if i := strings.Index(host, "%"); i > -1 {
|
||||
host = host[:i]
|
||||
}
|
||||
|
||||
// Check whether port is a valid port number.
|
||||
p, err := strconv.Atoi(port)
|
||||
if err != nil {
|
||||
@@ -366,7 +403,7 @@ func CheckLocalServerAddr(serverAddr string) error {
|
||||
// 0.0.0.0 is a wildcard address and refers to local network
|
||||
// addresses. I.e, 0.0.0.0:9000 like ":9000" refers to port
|
||||
// 9000 on localhost.
|
||||
if host != "" && host != net.IPv4zero.String() {
|
||||
if host != "" && host != net.IPv4zero.String() && host != net.IPv6zero.String() {
|
||||
isLocalHost, err := isLocalHost(host)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
+14
-8
@@ -138,7 +138,7 @@ func TestGetHostIP(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
ipList, err := getHostIP4(testCase.host)
|
||||
ipList, err := getHostIP(testCase.host)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
@@ -157,17 +157,22 @@ func TestGetHostIP(t *testing.T) {
|
||||
|
||||
// Tests finalize api endpoints.
|
||||
func TestGetAPIEndpoints(t *testing.T) {
|
||||
host, port := globalMinioHost, globalMinioAddr
|
||||
defer func() {
|
||||
globalMinioHost, globalMinioAddr = host, port
|
||||
}()
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
host, port string
|
||||
expectedResult string
|
||||
}{
|
||||
{":80", "http://127.0.0.1:80"},
|
||||
{"127.0.0.1:80", "http://127.0.0.1:80"},
|
||||
{"localhost:80", "http://localhost:80"},
|
||||
{"", "80", "http://127.0.0.1:80"},
|
||||
{"127.0.0.1", "80", "http://127.0.0.1:80"},
|
||||
{"localhost", "80", "http://localhost:80"},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
apiEndpoints := getAPIEndpoints(testCase.serverAddr)
|
||||
globalMinioHost, globalMinioPort = testCase.host, testCase.port
|
||||
apiEndpoints := getAPIEndpoints()
|
||||
apiEndpointSet := set.CreateStringSet(apiEndpoints...)
|
||||
if !apiEndpointSet.Contains(testCase.expectedResult) {
|
||||
t.Fatalf("test %d: expected: Found, got: Not Found", i+1)
|
||||
@@ -330,7 +335,7 @@ func TestSameLocalAddrs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestIsHostIPv4(t *testing.T) {
|
||||
func TestIsHostIP(t *testing.T) {
|
||||
testCases := []struct {
|
||||
args string
|
||||
expectedResult bool
|
||||
@@ -341,10 +346,11 @@ func TestIsHostIPv4(t *testing.T) {
|
||||
{"http://192.168.1.0", false},
|
||||
{"http://192.168.1.0:9000", false},
|
||||
{"192.168.1.0", true},
|
||||
{"[2001:3984:3989::20%eth0]:9000", true},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
ret := isHostIPv4(testCase.args)
|
||||
ret := isHostIP(testCase.args)
|
||||
if testCase.expectedResult != ret {
|
||||
t.Fatalf("expected: %v , got: %v", testCase.expectedResult, ret)
|
||||
}
|
||||
|
||||
+65
-3
@@ -22,6 +22,7 @@ import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"strings"
|
||||
@@ -89,6 +90,60 @@ func (sys *NotificationSys) DeleteBucket(ctx context.Context, bucketName string)
|
||||
}()
|
||||
}
|
||||
|
||||
// ReloadFormat - calls ReloadFormat RPC call on all peers.
|
||||
func (sys *NotificationSys) ReloadFormat(dryRun bool) map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
var wg sync.WaitGroup
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to load format in three attempts, before giving up.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.ReloadFormat(dryRun)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
errors[addr] = err
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
}(addr, client)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
// LoadUsers - calls LoadUsers RPC call on all peers.
|
||||
func (sys *NotificationSys) LoadUsers() map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
var wg sync.WaitGroup
|
||||
for addr, client := range sys.peerRPCClientMap {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to load users in three attempts.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.LoadUsers()
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
errors[addr] = err
|
||||
// Wait for one second and no need wait after last attempt.
|
||||
if i < 2 {
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
}
|
||||
}(addr, client)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
return errors
|
||||
}
|
||||
|
||||
// LoadCredentials - calls LoadCredentials RPC call on all peers.
|
||||
func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
|
||||
errors := make(map[xnet.Host]error)
|
||||
@@ -97,7 +152,7 @@ func (sys *NotificationSys) LoadCredentials() map[xnet.Host]error {
|
||||
wg.Add(1)
|
||||
go func(addr xnet.Host, client *PeerRPCClient) {
|
||||
defer wg.Done()
|
||||
// Try to set credentials in three attempts.
|
||||
// Try to load credentials in three attempts.
|
||||
for i := 0; i < 3; i++ {
|
||||
err := client.LoadCredentials()
|
||||
if err == nil {
|
||||
@@ -488,10 +543,10 @@ func (args eventArgs) ToEvent() event.Event {
|
||||
host := globalMinioHost
|
||||
if host == "" {
|
||||
// FIXME: Send FQDN or hostname of this machine than sending IP address.
|
||||
host = localIP4.ToSlice()[0]
|
||||
host = sortIPs(localIP4.ToSlice())[0]
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s://%s:%s", getURLScheme(globalIsSSL), host, globalMinioPort)
|
||||
return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
|
||||
}
|
||||
|
||||
eventTime := UTCNow()
|
||||
@@ -501,6 +556,10 @@ func (args eventArgs) ToEvent() event.Event {
|
||||
"x-amz-request-id": args.RespElements["requestId"],
|
||||
"x-minio-origin-endpoint": getOriginEndpoint(), // Minio specific custom elements.
|
||||
}
|
||||
// Add deployment as part of
|
||||
if globalDeploymentID != "" {
|
||||
respElements["x-minio-deployment-id"] = globalDeploymentID
|
||||
}
|
||||
if args.RespElements["content-length"] != "" {
|
||||
respElements["content-length"] = args.RespElements["content-length"]
|
||||
}
|
||||
@@ -537,6 +596,9 @@ func (args eventArgs) ToEvent() event.Event {
|
||||
if args.EventName != event.ObjectRemovedDelete {
|
||||
newEvent.S3.Object.ETag = args.Object.ETag
|
||||
newEvent.S3.Object.Size = args.Object.Size
|
||||
if args.Object.IsCompressed() {
|
||||
newEvent.S3.Object.Size = args.Object.GetActualSize()
|
||||
}
|
||||
newEvent.S3.Object.ContentType = args.Object.ContentType
|
||||
newEvent.S3.Object.UserMetadata = args.Object.UserDefined
|
||||
}
|
||||
|
||||
@@ -47,7 +47,7 @@ func toObjectErr(err error, params ...string) error {
|
||||
Object: params[1],
|
||||
}
|
||||
}
|
||||
case errIsNotRegular, errFileAccessDenied:
|
||||
case errIsNotRegular:
|
||||
if len(params) >= 2 {
|
||||
err = ObjectExistsAsDirectory{
|
||||
Bucket: params[0],
|
||||
|
||||
@@ -94,6 +94,7 @@ type ObjectLayer interface {
|
||||
|
||||
// Supported operations check
|
||||
IsNotificationSupported() bool
|
||||
IsListenBucketSupported() bool
|
||||
IsEncryptionSupported() bool
|
||||
|
||||
// Compression support check.
|
||||
|
||||
+15
-9
@@ -23,6 +23,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net"
|
||||
"net/http"
|
||||
"path"
|
||||
"runtime"
|
||||
@@ -299,11 +300,11 @@ func getHostsSlice(records []dns.SrvRecord) []string {
|
||||
return hosts
|
||||
}
|
||||
|
||||
// returns a random host (and corresponding port) from a slice of DNS records
|
||||
func getRandomHostPort(records []dns.SrvRecord) (string, int) {
|
||||
// returns a host (and corresponding port) from a slice of DNS records
|
||||
func getHostFromSrv(records []dns.SrvRecord) string {
|
||||
rand.Seed(time.Now().Unix())
|
||||
srvRecord := records[rand.Intn(len(records))]
|
||||
return srvRecord.Host, srvRecord.Port
|
||||
return net.JoinHostPort(srvRecord.Host, fmt.Sprintf("%d", srvRecord.Port))
|
||||
}
|
||||
|
||||
// IsCompressed returns true if the object is marked as compressed.
|
||||
@@ -502,6 +503,9 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, cleanUpFns ...func())
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Decrypt the ETag before top layer consumes this value.
|
||||
oi.ETag = getDecryptedETag(h, oi, copySource)
|
||||
|
||||
// Apply the skipLen and limit on the
|
||||
// decrypted stream
|
||||
decReader = io.LimitReader(ioutil.NewSkipReader(decReader, skipLen), decRangeLength)
|
||||
@@ -635,12 +639,13 @@ func (p *PutObjReader) MD5CurrentHexString() string {
|
||||
func NewPutObjReader(rawReader *hash.Reader, encReader *hash.Reader, encKey []byte) *PutObjReader {
|
||||
p := PutObjReader{Reader: rawReader, rawReader: rawReader}
|
||||
|
||||
var objKey crypto.ObjectKey
|
||||
copy(objKey[:], encKey)
|
||||
p.sealMD5Fn = sealETagFn(objKey)
|
||||
if encReader != nil {
|
||||
if len(encKey) != 0 && encReader != nil {
|
||||
var objKey crypto.ObjectKey
|
||||
copy(objKey[:], encKey)
|
||||
p.sealMD5Fn = sealETagFn(objKey)
|
||||
p.Reader = encReader
|
||||
}
|
||||
|
||||
return &p
|
||||
}
|
||||
|
||||
@@ -651,9 +656,10 @@ func sealETag(encKey crypto.ObjectKey, md5CurrSum []byte) []byte {
|
||||
}
|
||||
return encKey.SealETag(md5CurrSum)
|
||||
}
|
||||
|
||||
func sealETagFn(key crypto.ObjectKey) SealMD5CurrFn {
|
||||
fn1 := func(md5sumcurr []byte) []byte {
|
||||
fn := func(md5sumcurr []byte) []byte {
|
||||
return sealETag(key, md5sumcurr)
|
||||
}
|
||||
return fn1
|
||||
return fn
|
||||
}
|
||||
|
||||
@@ -75,7 +75,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if !ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
// If the object is not modified since the specified time.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -89,7 +89,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
// If the object is modified since the specified time.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if objInfo.ETag != "" && !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
|
||||
// If the object ETag does not match with the specified ETag.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func checkCopyObjectPreconditions(w http.ResponseWriter, r *http.Request, objInf
|
||||
if objInfo.ETag != "" && isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) {
|
||||
// If the object ETag matches with the specified ETag.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -174,7 +174,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
|
||||
if ifModifiedSince(objInfo.ModTime, givenTime) {
|
||||
// If the object is modified since the specified time.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -187,7 +187,7 @@ func checkPreconditions(w http.ResponseWriter, r *http.Request, objInfo ObjectIn
|
||||
if !isETagEqual(objInfo.ETag, ifMatchETagHeader) {
|
||||
// If the object ETag does not match with the specified ETag.
|
||||
writeHeaders()
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL)
|
||||
writeErrorResponse(w, ErrPreconditionFailed, r.URL, guessIsBrowserReq(r))
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
+362
-249
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,7 @@ import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -1811,9 +1812,13 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
|
||||
// this is required even to assert the copied object,
|
||||
bytesData := []struct {
|
||||
byteData []byte
|
||||
md5sum string
|
||||
}{
|
||||
{generateBytesData(6 * humanize.KiByte)},
|
||||
{byteData: generateBytesData(6 * humanize.KiByte)},
|
||||
}
|
||||
h := md5.New()
|
||||
h.Write(bytesData[0].byteData)
|
||||
bytesData[0].md5sum = hex.EncodeToString(h.Sum(nil))
|
||||
|
||||
buffers := []*bytes.Buffer{
|
||||
new(bytes.Buffer),
|
||||
@@ -1826,23 +1831,29 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
|
||||
objectName string
|
||||
contentLength int64
|
||||
textData []byte
|
||||
md5sum string
|
||||
metaData map[string]string
|
||||
}{
|
||||
// case - 1.
|
||||
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
|
||||
{bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)},
|
||||
|
||||
// case - 2.
|
||||
// used for anonymous HTTP request test.
|
||||
{bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)},
|
||||
{bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)},
|
||||
}
|
||||
|
||||
// iterate through the above set of inputs and upload the object.
|
||||
for i, input := range putObjectInputs {
|
||||
// uploading the object.
|
||||
_, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewBuffer(input.textData), input.contentLength, input.metaData[""], ""), input.metaData, opts)
|
||||
// if object upload fails stop the test.
|
||||
if err != nil {
|
||||
t.Fatalf("Put Object case %d: Error uploading object: <ERROR> %v", i+1, err)
|
||||
}
|
||||
if objInfo.ETag != input.md5sum {
|
||||
t.Fatalf("Put Object case %d: Checksum mismatched: <ERROR> got %s, expected %s", i+1, input.md5sum, objInfo.ETag)
|
||||
}
|
||||
}
|
||||
|
||||
// test cases with inputs and expected result for Copy Object.
|
||||
@@ -2105,6 +2116,16 @@ func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string,
|
||||
t.Fatalf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
|
||||
}
|
||||
if rec.Code == http.StatusOK {
|
||||
var cpObjResp CopyObjectResponse
|
||||
if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil {
|
||||
t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: <ERROR> %s", i+1, instanceType, err)
|
||||
}
|
||||
|
||||
retag := canonicalizeETag(cpObjResp.ETag)
|
||||
if retag != bytesData[0].md5sum {
|
||||
t.Fatalf("Test %d: %s: Failed to CopyObject: <ERROR> got %s, expected %s", i+1, instanceType, retag, bytesData[0].md5sum)
|
||||
}
|
||||
|
||||
// See if the new object is formed.
|
||||
// testing whether the copy was successful.
|
||||
err = obj.GetObject(context.Background(), testCase.bucketName, testCase.newObjectName, 0, int64(len(bytesData[0].byteData)), buffers[0], "", opts)
|
||||
|
||||
+19
-1
@@ -115,12 +115,30 @@ func (rpcClient *PeerRPCClient) SendEvent(bucketName string, targetID, remoteTar
|
||||
return err
|
||||
}
|
||||
|
||||
// ReloadFormat - calls reload format RPC.
|
||||
func (rpcClient *PeerRPCClient) ReloadFormat(dryRun bool) error {
|
||||
args := ReloadFormatArgs{
|
||||
DryRun: dryRun,
|
||||
}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(peerServiceName+".ReloadFormat", &args, &reply)
|
||||
}
|
||||
|
||||
// LoadUsers - calls load users RPC.
|
||||
func (rpcClient *PeerRPCClient) LoadUsers() error {
|
||||
args := AuthArgs{}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(peerServiceName+".LoadUsers", &args, &reply)
|
||||
}
|
||||
|
||||
// LoadCredentials - calls load credentials RPC.
|
||||
func (rpcClient *PeerRPCClient) LoadCredentials() error {
|
||||
args := AuthArgs{}
|
||||
reply := VoidReply{}
|
||||
|
||||
return rpcClient.Call(peerServiceName+".SetCredentials", &args, &reply)
|
||||
return rpcClient.Call(peerServiceName+".LoadCredentials", &args, &reply)
|
||||
}
|
||||
|
||||
// NewPeerRPCClient - returns new peer RPC client.
|
||||
|
||||
@@ -192,6 +192,30 @@ func (receiver *peerRPCReceiver) SendEvent(args *SendEventArgs, reply *bool) err
|
||||
return nil
|
||||
}
|
||||
|
||||
// ReloadFormatArgs - send event RPC arguments.
|
||||
type ReloadFormatArgs struct {
|
||||
AuthArgs
|
||||
DryRun bool
|
||||
}
|
||||
|
||||
// ReloadFormat - handles reload format RPC call, reloads latest `format.json`
|
||||
func (receiver *peerRPCReceiver) ReloadFormat(args *ReloadFormatArgs, reply *VoidReply) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
return objAPI.ReloadFormat(context.Background(), args.DryRun)
|
||||
}
|
||||
|
||||
// LoadUsers - handles load users RPC call.
|
||||
func (receiver *peerRPCReceiver) LoadUsers(args *AuthArgs, reply *VoidReply) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
return globalIAMSys.Load(objAPI)
|
||||
}
|
||||
|
||||
// LoadCredentials - handles load credentials RPC call.
|
||||
func (receiver *peerRPCReceiver) LoadCredentials(args *AuthArgs, reply *VoidReply) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// +build plan9
|
||||
// +build plan9 solaris
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
@@ -1,4 +1,4 @@
|
||||
// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris
|
||||
// +build darwin dragonfly freebsd linux nacl netbsd openbsd
|
||||
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
|
||||
+24
-4
@@ -71,6 +71,7 @@ type posix struct {
|
||||
connected bool
|
||||
|
||||
diskMount bool // indicates if the path is an actual mount.
|
||||
driveSync bool // indicates if the backend is synchronous.
|
||||
|
||||
// Disk usage metrics
|
||||
stopUsageCh chan struct{}
|
||||
@@ -83,8 +84,10 @@ func checkPathLength(pathName string) error {
|
||||
return errFileNameTooLong
|
||||
}
|
||||
|
||||
// Convert any '\' to '/'.
|
||||
pathName = filepath.ToSlash(pathName)
|
||||
if runtime.GOOS == "windows" {
|
||||
// Convert any '\' to '/'.
|
||||
pathName = filepath.ToSlash(pathName)
|
||||
}
|
||||
|
||||
// Check each path segment length is > 255
|
||||
for len(pathName) > 0 && pathName != "." && pathName != "/" {
|
||||
@@ -189,6 +192,15 @@ func newPosix(path string) (*posix, error) {
|
||||
diskMount: mountinfo.IsLikelyMountPoint(path),
|
||||
}
|
||||
|
||||
var pf BoolFlag
|
||||
if driveSync := os.Getenv("MINIO_DRIVE_SYNC"); driveSync != "" {
|
||||
pf, err = ParseBoolFlag(driveSync)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.driveSync = bool(pf)
|
||||
}
|
||||
|
||||
if !p.diskMount {
|
||||
go p.diskUsage(globalServiceDoneCh)
|
||||
}
|
||||
@@ -374,6 +386,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
default:
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
atomic.AddUint64(&s.totalUsed, uint64(fi.Size()))
|
||||
@@ -413,6 +426,7 @@ func (s *posix) diskUsage(doneCh chan struct{}) {
|
||||
default:
|
||||
fi, err := os.Stat(entry)
|
||||
if err != nil {
|
||||
err = osErrToFSFileErr(err)
|
||||
return err
|
||||
}
|
||||
usage = usage + uint64(fi.Size())
|
||||
@@ -1009,8 +1023,14 @@ func (s *posix) AppendFile(volume, path string, buf []byte) (err error) {
|
||||
return errFaultyDisk
|
||||
}
|
||||
|
||||
// Create file if not found
|
||||
w, err := s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
|
||||
var w *os.File
|
||||
// Create file if not found, additionally also enables synchronous
|
||||
// operation if asked by the user.
|
||||
if s.driveSync {
|
||||
w, err = s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_APPEND|os.O_WRONLY)
|
||||
} else {
|
||||
w, err = s.openFile(volume, path, os.O_CREATE|os.O_APPEND|os.O_WRONLY)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
+10
-10
@@ -128,6 +128,15 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
|
||||
}
|
||||
defer closeStorageDisks(storageDisks)
|
||||
|
||||
// Attempt to load all `format.json` from all disks.
|
||||
formatConfigs, sErrs := loadFormatXLAll(storageDisks)
|
||||
// Check if we have
|
||||
for i, sErr := range sErrs {
|
||||
if _, ok := formatCriticalErrors[sErr]; ok {
|
||||
return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Connect to all storage disks, a connection failure will be
|
||||
// only logged after some retries.
|
||||
for _, disk := range storageDisks {
|
||||
@@ -139,15 +148,6 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to load all `format.json` from all disks.
|
||||
formatConfigs, sErrs := loadFormatXLAll(storageDisks)
|
||||
// Check if we have
|
||||
for i, sErr := range sErrs {
|
||||
if _, ok := formatCriticalErrors[sErr]; ok {
|
||||
return nil, fmt.Errorf("Disk %s: %s", endpoints[i], sErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Pre-emptively check if one of the formatted disks
|
||||
// is invalid. This function returns success for the
|
||||
// most part unless one of the formats is not consistent
|
||||
@@ -210,7 +210,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints EndpointLi
|
||||
}
|
||||
}
|
||||
|
||||
logger.SetDeploymentID(format.ID)
|
||||
globalDeploymentID = format.ID
|
||||
|
||||
if err = formatXLFixLocalDeploymentID(context.Background(), storageDisks, format); err != nil {
|
||||
return nil, err
|
||||
|
||||
+30
-21
@@ -35,9 +35,10 @@ const DefaultRESTTimeout = 1 * time.Minute
|
||||
|
||||
// Client - http based RPC client.
|
||||
type Client struct {
|
||||
httpClient *http.Client
|
||||
url *url.URL
|
||||
newAuthToken func() string
|
||||
httpClient *http.Client
|
||||
httpIdleConnsCloser func()
|
||||
url *url.URL
|
||||
newAuthToken func() string
|
||||
}
|
||||
|
||||
// Call - make a REST call.
|
||||
@@ -67,6 +68,13 @@ func (c *Client) Call(method string, values url.Values, body io.Reader) (reply i
|
||||
return resp.Body, nil
|
||||
}
|
||||
|
||||
// Close closes all idle connections of the underlying http client
|
||||
func (c *Client) Close() {
|
||||
if c.httpIdleConnsCloser != nil {
|
||||
c.httpIdleConnsCloser()
|
||||
}
|
||||
}
|
||||
|
||||
func newCustomDialContext(timeout time.Duration) func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
return func(ctx context.Context, network, addr string) (net.Conn, error) {
|
||||
dialer := &net.Dialer{
|
||||
@@ -84,25 +92,26 @@ func newCustomDialContext(timeout time.Duration) func(ctx context.Context, netwo
|
||||
}
|
||||
}
|
||||
|
||||
// NewClient - returns new RPC client.
|
||||
// NewClient - returns new REST client.
|
||||
func NewClient(url *url.URL, tlsConfig *tls.Config, timeout time.Duration, newAuthToken func() string) *Client {
|
||||
// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
|
||||
// except custom DialContext and TLSClientConfig.
|
||||
tr := &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: newCustomDialContext(timeout),
|
||||
MaxIdleConnsPerHost: 4096,
|
||||
MaxIdleConns: 4096,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
TLSClientConfig: tlsConfig,
|
||||
DisableCompression: true,
|
||||
}
|
||||
|
||||
return &Client{
|
||||
httpClient: &http.Client{
|
||||
// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
|
||||
// except custom DialContext and TLSClientConfig.
|
||||
Transport: &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: newCustomDialContext(timeout),
|
||||
MaxIdleConnsPerHost: 4096,
|
||||
MaxIdleConns: 4096,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
TLSClientConfig: tlsConfig,
|
||||
DisableCompression: true,
|
||||
},
|
||||
},
|
||||
url: url,
|
||||
newAuthToken: newAuthToken,
|
||||
httpClient: &http.Client{Transport: tr},
|
||||
httpIdleConnsCloser: tr.CloseIdleConnections,
|
||||
url: url,
|
||||
newAuthToken: newAuthToken,
|
||||
}
|
||||
}
|
||||
|
||||
+4
-4
@@ -47,8 +47,8 @@ func registerDistXLRouters(router *mux.Router, endpoints EndpointList) {
|
||||
|
||||
// List of some generic handlers which are applied for all incoming requests.
|
||||
var globalHandlers = []HandlerFunc{
|
||||
// set x-amz-request-id header.
|
||||
addrequestIDHeader,
|
||||
// set x-amz-request-id, x-minio-deployment-id header.
|
||||
addCustomHeaders,
|
||||
// set HTTP security headers such as Content-Security-Policy.
|
||||
addSecurityHeaders,
|
||||
// Forward path style requests to actual host in a bucket federated setup.
|
||||
@@ -107,8 +107,8 @@ func configureServerHandler(endpoints EndpointList) (http.Handler, error) {
|
||||
// Add Admin RPC router
|
||||
registerAdminRPCRouter(router)
|
||||
|
||||
// Add Admin router.
|
||||
registerAdminRouter(router)
|
||||
// Add Admin router, all APIs are enabled in server mode.
|
||||
registerAdminRouter(router, true)
|
||||
|
||||
// Add healthcheck router
|
||||
registerHealthCheckRouter(router)
|
||||
|
||||
+15
-27
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -40,14 +41,14 @@ func init() {
|
||||
var serverFlags = []cli.Flag{
|
||||
cli.StringFlag{
|
||||
Name: "address",
|
||||
Value: ":" + globalMinioPort,
|
||||
Usage: "Bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname.",
|
||||
Value: ":" + globalMinioDefaultPort,
|
||||
Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
|
||||
},
|
||||
}
|
||||
|
||||
var serverCmd = cli.Command{
|
||||
Name: "server",
|
||||
Usage: "Start object storage server.",
|
||||
Usage: "start object storage server",
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
Action: serverMain,
|
||||
CustomHelpTemplate: `NAME:
|
||||
@@ -112,10 +113,10 @@ EXAMPLES:
|
||||
4. Start erasure coded minio server on a node with 64 drives.
|
||||
$ {{.HelpName}} /mnt/export{1...64}
|
||||
|
||||
5. Start distributed minio server on an 8 node setup with 8 drives each. Run following command on all the 8 nodes.
|
||||
5. Start distributed minio server on an 32 node setup with 32 drives each. Run following command on all the 32 nodes.
|
||||
$ export MINIO_ACCESS_KEY=minio
|
||||
$ export MINIO_SECRET_KEY=miniostorage
|
||||
$ {{.HelpName}} http://node{1...8}.example.com/mnt/export/{1...8}
|
||||
$ {{.HelpName}} http://node{1...32}.example.com/mnt/export/{1...32}
|
||||
|
||||
6. Start minio server with edge caching enabled.
|
||||
$ export MINIO_CACHE_DRIVES="/mnt/drive1;/mnt/drive2;/mnt/drive3;/mnt/drive4"
|
||||
@@ -147,9 +148,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
|
||||
// Handle common command args.
|
||||
handleCommonCmdArgs(ctx)
|
||||
|
||||
// Server address.
|
||||
serverAddr := ctx.String("address")
|
||||
logger.FatalIf(CheckLocalServerAddr(serverAddr), "Unable to validate passed arguments")
|
||||
logger.FatalIf(CheckLocalServerAddr(globalCLIContext.Addr), "Unable to validate passed arguments")
|
||||
|
||||
var setupType SetupType
|
||||
var err error
|
||||
@@ -162,9 +161,9 @@ func serverHandleCmdArgs(ctx *cli.Context) {
|
||||
|
||||
endpoints := strings.Fields(os.Getenv("MINIO_ENDPOINTS"))
|
||||
if len(endpoints) > 0 {
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, endpoints...)
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, endpoints...)
|
||||
} else {
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(serverAddr, ctx.Args()...)
|
||||
globalMinioAddr, globalEndpoints, setupType, globalXLSetCount, globalXLSetDriveCount, err = createServerEndpoints(globalCLIContext.Addr, ctx.Args()...)
|
||||
}
|
||||
logger.FatalIf(err, "Invalid command line arguments")
|
||||
|
||||
@@ -205,19 +204,6 @@ func serverMain(ctx *cli.Context) {
|
||||
// error during initialization will be shown as a fatal message
|
||||
logger.Disable = true
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
jsonFlag := ctx.IsSet("json") || ctx.GlobalIsSet("json")
|
||||
if jsonFlag {
|
||||
logger.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet")
|
||||
if quietFlag {
|
||||
logger.EnableQuiet()
|
||||
}
|
||||
|
||||
// Handle all server command args.
|
||||
serverHandleCmdArgs(ctx)
|
||||
|
||||
@@ -246,7 +232,7 @@ func serverMain(ctx *cli.Context) {
|
||||
}
|
||||
}
|
||||
|
||||
if !quietFlag {
|
||||
if !globalCLIContext.Quiet {
|
||||
// Check for new updates from dl.minio.io.
|
||||
mode := globalMinioModeFS
|
||||
if globalIsDistXL {
|
||||
@@ -381,7 +367,10 @@ func serverMain(ctx *cli.Context) {
|
||||
|
||||
// Initialize notification system.
|
||||
if err = globalNotificationSys.Init(newObject); err != nil {
|
||||
logger.Fatal(err, "Unable to initialize notification system")
|
||||
logger.LogIf(context.Background(), err)
|
||||
}
|
||||
if globalAutoEncryption && !newObject.IsEncryptionSupported() {
|
||||
logger.Fatal(errors.New("Invalid KMS configuration"), "auto-encryption is enabled but server does not support encryption")
|
||||
}
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
@@ -389,8 +378,7 @@ func serverMain(ctx *cli.Context) {
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Prints the formatted startup message once object layer is initialized.
|
||||
apiEndpoints := getAPIEndpoints(globalMinioAddr)
|
||||
printStartupMessage(apiEndpoints)
|
||||
printStartupMessage(getAPIEndpoints())
|
||||
|
||||
// Set uptime time after object layer has initialized.
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
+21
-14
@@ -20,12 +20,13 @@ import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"net"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
)
|
||||
|
||||
// Documentation links, these are part of message printing code.
|
||||
@@ -76,6 +77,19 @@ func printStartupMessage(apiEndPoints []string) {
|
||||
}
|
||||
}
|
||||
|
||||
// Returns true if input is not IPv4, false if it is.
|
||||
func isNotIPv4(host string) bool {
|
||||
h, _, err := net.SplitHostPort(host)
|
||||
if err != nil {
|
||||
h = host
|
||||
}
|
||||
ip := net.ParseIP(h)
|
||||
ok := ip.To4() != nil // This is always true of IP is IPv4
|
||||
|
||||
// Returns true if input is not IPv4.
|
||||
return !ok
|
||||
}
|
||||
|
||||
// strip api endpoints list with standard ports such as
|
||||
// port "80" and "443" before displaying on the startup
|
||||
// banner. Returns a new list of API endpoints.
|
||||
@@ -83,23 +97,16 @@ func stripStandardPorts(apiEndpoints []string) (newAPIEndpoints []string) {
|
||||
newAPIEndpoints = make([]string, len(apiEndpoints))
|
||||
// Check all API endpoints for standard ports and strip them.
|
||||
for i, apiEndpoint := range apiEndpoints {
|
||||
url, err := url.Parse(apiEndpoint)
|
||||
u, err := xnet.ParseURL(apiEndpoint)
|
||||
if err != nil {
|
||||
newAPIEndpoints[i] = apiEndpoint
|
||||
continue
|
||||
}
|
||||
host, port := mustSplitHostPort(url.Host)
|
||||
// For standard HTTP(s) ports such as "80" and "443"
|
||||
// apiEndpoints should only be host without port.
|
||||
switch {
|
||||
case url.Scheme == "http" && port == "80":
|
||||
fallthrough
|
||||
case url.Scheme == "https" && port == "443":
|
||||
url.Host = host
|
||||
newAPIEndpoints[i] = url.String()
|
||||
default:
|
||||
newAPIEndpoints[i] = apiEndpoint
|
||||
if globalMinioHost == "" && isNotIPv4(u.Host) {
|
||||
// Skip all non-IPv4 endpoints when we bind to all interfaces.
|
||||
continue
|
||||
}
|
||||
newAPIEndpoints[i] = u.String()
|
||||
}
|
||||
return newAPIEndpoints
|
||||
}
|
||||
@@ -116,7 +123,7 @@ func printServerCommonMsg(apiEndpoints []string) {
|
||||
|
||||
// Colorize the message and print.
|
||||
logger.StartupMessage(colorBlue("Endpoint: ") + colorBold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
if isTerminal() {
|
||||
if isTerminal() && !globalCLIContext.Anonymous {
|
||||
logger.StartupMessage(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logger.StartupMessage(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
if region != "" {
|
||||
|
||||
@@ -28,6 +28,9 @@ const (
|
||||
|
||||
// DistXLSetupType - Distributed XL setup type enum.
|
||||
DistXLSetupType
|
||||
|
||||
// GatewaySetupType - gateway setup type enum.
|
||||
GatewaySetupType
|
||||
)
|
||||
|
||||
func (setupType SetupType) String() string {
|
||||
@@ -38,6 +41,8 @@ func (setupType SetupType) String() string {
|
||||
return globalMinioModeXL
|
||||
case DistXLSetupType:
|
||||
return globalMinioModeDistXL
|
||||
case GatewaySetupType:
|
||||
return globalMinioModeGatewayPrefix
|
||||
}
|
||||
|
||||
return ""
|
||||
|
||||
@@ -322,6 +322,7 @@ func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPa
|
||||
// Close - marks the client as closed.
|
||||
func (client *storageRESTClient) Close() error {
|
||||
client.connected = false
|
||||
client.restClient.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -377,7 +377,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpoints EndpointList) {
|
||||
Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
|
||||
subrouter.Methods(http.MethodPost).Path("/" + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler)).
|
||||
Queries(restQueries(storageRESTSrcVolume, storageRESTSrcPath, storageRESTDstVolume, storageRESTDstPath)...)
|
||||
|
||||
subrouter.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
}
|
||||
|
||||
router.NotFoundHandler = http.HandlerFunc(httpTraceAll(notFoundHandler))
|
||||
}
|
||||
|
||||
@@ -127,6 +127,8 @@ type ClientGrantsResult struct {
|
||||
func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := newContext(r, w, "AssumeRoleWithClientGrants")
|
||||
|
||||
defer logger.AuditLog(w, r, "AssumeRoleWithClientGrants", nil)
|
||||
|
||||
if globalIAMValidators == nil {
|
||||
writeSTSErrorResponse(w, ErrSTSNotInitialized)
|
||||
return
|
||||
|
||||
@@ -2319,7 +2319,7 @@ func getEndpointsLocalAddr(endpoints EndpointList) string {
|
||||
}
|
||||
}
|
||||
|
||||
return globalMinioHost + ":" + globalMinioPort
|
||||
return net.JoinHostPort(globalMinioHost, globalMinioPort)
|
||||
}
|
||||
|
||||
// fetches a random number between range min-max.
|
||||
|
||||
+2
-2
@@ -40,12 +40,12 @@ import (
|
||||
// Check for new software updates.
|
||||
var updateCmd = cli.Command{
|
||||
Name: "update",
|
||||
Usage: "Check for a new software update.",
|
||||
Usage: "update minio to latest release",
|
||||
Action: mainUpdate,
|
||||
Flags: []cli.Flag{
|
||||
cli.BoolFlag{
|
||||
Name: "quiet",
|
||||
Usage: "Disable any update prompt message.",
|
||||
Usage: "disable any update prompt message",
|
||||
},
|
||||
},
|
||||
CustomHelpTemplate: `Name:
|
||||
|
||||
+15
-6
@@ -105,6 +105,12 @@ const (
|
||||
httpsScheme = "https"
|
||||
)
|
||||
|
||||
// nopCharsetConverter is a dummy charset convert which just copies input to output,
|
||||
// it is used to ignore custom encoding charset in S3 XML body.
|
||||
func nopCharsetConverter(label string, input io.Reader) (io.Reader, error) {
|
||||
return input, nil
|
||||
}
|
||||
|
||||
// xmlDecoder provide decoded value in xml.
|
||||
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
||||
var lbody io.Reader
|
||||
@@ -114,6 +120,8 @@ func xmlDecoder(body io.Reader, v interface{}, size int64) error {
|
||||
lbody = body
|
||||
}
|
||||
d := xml.NewDecoder(lbody)
|
||||
// Ignore any encoding set in the XML body
|
||||
d.CharsetReader = nopCharsetConverter
|
||||
return d.Decode(v)
|
||||
}
|
||||
|
||||
@@ -397,12 +405,13 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
|
||||
object = prefix
|
||||
}
|
||||
reqInfo := &logger.ReqInfo{
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
RemoteHost: handlers.GetSourceIP(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
API: api,
|
||||
BucketName: bucket,
|
||||
ObjectName: object,
|
||||
DeploymentID: w.Header().Get(responseDeploymentIDKey),
|
||||
RequestID: w.Header().Get(responseRequestIDKey),
|
||||
RemoteHost: handlers.GetSourceIP(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
API: api,
|
||||
BucketName: bucket,
|
||||
ObjectName: object,
|
||||
}
|
||||
return logger.SetReqInfo(context.Background(), reqInfo)
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user