mirror of
https://github.com/pgsty/minio.git
synced 2026-07-31 07:45:19 +03:00
Compare commits
222 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b9f622824a | |||
| 11c4223f2c | |||
| 26903da8c2 | |||
| 353f2d3a6e | |||
| c8947af227 | |||
| 075b8903d7 | |||
| b8463a738c | |||
| 6f4862659f | |||
| f59d7a04b4 | |||
| b28d5fa633 | |||
| 48dbd49980 | |||
| 3dfe254a11 | |||
| ec2920e981 | |||
| 0f5483f497 | |||
| 8a6b0cc0cd | |||
| 2c56788f8d | |||
| 145328ac9f | |||
| 45a568dd85 | |||
| 7dcc1e92b4 | |||
| 999ae1cb96 | |||
| 6651c2fc5f | |||
| 976870a391 | |||
| f99987e47c | |||
| 1c3f244fc5 | |||
| 986aa8fabf | |||
| 2559614bfd | |||
| a4d1ef1b62 | |||
| 432bf7d99e | |||
| 18c4e5d357 | |||
| 64f4dbc272 | |||
| 9ba57a8df0 | |||
| 5621e6a494 | |||
| 0f0758aece | |||
| a0e02f43e1 | |||
| 28352f3f5d | |||
| 975972d57e | |||
| 458f22f37c | |||
| 0bba3cc8e3 | |||
| bac5303b10 | |||
| e01b2fc06d | |||
| 072fcf3ba6 | |||
| b78f6fbcc5 | |||
| 99ca8a2928 | |||
| 9b3dd44607 | |||
| 3c5db69ffd | |||
| 28c26a9e59 | |||
| 9136734c02 | |||
| 348aa6566c | |||
| 9d98bf1c0f | |||
| a767ad321a | |||
| 1886d94e95 | |||
| 542f7ae42c | |||
| 59b3e0b79b | |||
| 8975da4e84 | |||
| cae4683971 | |||
| 5db1e9f3dd | |||
| 465274cd21 | |||
| 87fb911d38 | |||
| 155a90403a | |||
| c63afabc9b | |||
| 0404e747cf | |||
| 3bc9e6101c | |||
| f2ed149714 | |||
| 5a16dcf4cf | |||
| fa3d5d0f46 | |||
| bb292e4e38 | |||
| d1971b9a4d | |||
| fa3f6d75b6 | |||
| 298b470f69 | |||
| fc774957fe | |||
| c6258f5e97 | |||
| 85bc6003e9 | |||
| 0d9de50e21 | |||
| d13aa1c42d | |||
| 610dbe3479 | |||
| 2df1e2e9a9 | |||
| e372b5ed67 | |||
| 76f4f20609 | |||
| 99ddd35343 | |||
| 01e9adc4b3 | |||
| 4c63fd06c6 | |||
| 4f61bd025d | |||
| 40985cc4e3 | |||
| bb4efbf258 | |||
| a89c7299d1 | |||
| df027a8f51 | |||
| a02575ebf9 | |||
| 972a527b66 | |||
| 02910725c5 | |||
| 0ea8bfaf78 | |||
| 069cf9e8aa | |||
| 2121b78ea7 | |||
| 5016649f47 | |||
| 2b78444056 | |||
| 9b58a669e5 | |||
| e5b2e25caf | |||
| 4aa65910e5 | |||
| 8b272a3163 | |||
| f0b5c0ec7c | |||
| 44d53c9c67 | |||
| 6cf6828a4c | |||
| 01f04c717e | |||
| 0d32b22359 | |||
| cab298d68f | |||
| 1ea53b4d9f | |||
| d36dd80a8a | |||
| 3a539ce660 | |||
| 57c5c75611 | |||
| cf1fc45142 | |||
| 64c1c0f37d | |||
| 82857cd6df | |||
| 3cdc0c57c8 | |||
| dc365bca44 | |||
| 3b1626216d | |||
| 48aa2ac392 | |||
| 0d1e2ab509 | |||
| ebf4c447bb | |||
| 2b96d9f706 | |||
| 83abad0b37 | |||
| df346753e1 | |||
| f1d7780167 | |||
| 5a3c5aec31 | |||
| a4305742e8 | |||
| 640ebb2f79 | |||
| 402a5e3bea | |||
| f4dac979a2 | |||
| 3032f0f505 | |||
| 7765081db7 | |||
| 18bfe5cba6 | |||
| 14f0047295 | |||
| e6b2253da9 | |||
| ca64b86112 | |||
| 5f065e2a96 | |||
| a7afa469e2 | |||
| d103d5fb7c | |||
| 6683247080 | |||
| e4bd882f11 | |||
| 4448285a83 | |||
| 952c618441 | |||
| c5249c35d3 | |||
| de204a0a52 | |||
| 1b1b9e4801 | |||
| 1b0b2c1c76 | |||
| 7f5e037846 | |||
| 929a13f33f | |||
| a4209c10ea | |||
| b927523223 | |||
| 604417baf4 | |||
| 6e9ac8db59 | |||
| 0497d5c342 | |||
| 6b4f368dfe | |||
| 6507c30bbc | |||
| f44f2e341c | |||
| 60ea2b17ba | |||
| 5ed1a8ad23 | |||
| 27749c2124 | |||
| 1d99a560e3 | |||
| af82d27018 | |||
| f205689ff5 | |||
| 393c01d078 | |||
| e9037b9d36 | |||
| 4747adfcb4 | |||
| 52d8f564bf | |||
| 3fe33e7b15 | |||
| 4041e5f20d | |||
| 96c46c15e7 | |||
| 3bf67668b6 | |||
| e31e2c3bc2 | |||
| 214279aa57 | |||
| 4de6b15fca | |||
| 2bd694dbc8 | |||
| a2a8d54bb6 | |||
| 2040d32ef8 | |||
| f1015a5096 | |||
| 096427f973 | |||
| 5cec6bd80d | |||
| 6e9c91f43a | |||
| 2df8160f6a | |||
| 447fdd4097 | |||
| 28c5a887de | |||
| fbe8b3259d | |||
| e2aba9196f | |||
| 61b08137b0 | |||
| 1caad902cb | |||
| d99efa2c93 | |||
| 9ee83b89bb | |||
| a8cb43926a | |||
| b62cd8ed84 | |||
| 1b3a517683 | |||
| a9c0f1e0a4 | |||
| a099319e66 | |||
| 6e63904048 | |||
| 565ac4c861 | |||
| a4ecd8bca2 | |||
| 28eff0f6c1 | |||
| c27ece409b | |||
| 8c0ce2fee9 | |||
| d3cb79a57c | |||
| 11e15f9b4c | |||
| 4e92b2ecb8 | |||
| 80b83a51a3 | |||
| 777d12d928 | |||
| 607c8a9611 | |||
| 7f5a5b5e9d | |||
| d4ca2ee1a3 | |||
| 417ec0df56 | |||
| fbfb4fc5a0 | |||
| 97dc34fe06 | |||
| 9f9ba1e984 | |||
| 13c4ce3617 | |||
| 181e002c56 | |||
| 9d6e226692 | |||
| eb02261642 | |||
| 1396e91dd1 | |||
| 7ebf11b202 | |||
| 1c97dcb10a | |||
| 3a6111eff5 | |||
| 2463ae243a | |||
| c192e5c9b2 | |||
| d4eea224d4 | |||
| e55421ebdd | |||
| 0a5d57a91e |
@@ -26,8 +26,8 @@
|
||||
|
||||
## Your Environment
|
||||
<!--- Include as many relevant details about the environment you experienced the bug in -->
|
||||
* Version used:
|
||||
* Version used (`minio version`):
|
||||
* Environment name and version (e.g. nginx 1.9.1):
|
||||
* Server type and version:
|
||||
* Operating System and version:
|
||||
* Operating System and version (`uname -a`):
|
||||
* Link to your project:
|
||||
|
||||
@@ -18,6 +18,8 @@ env:
|
||||
script:
|
||||
## Run all the tests
|
||||
- make
|
||||
- diff -au <(gofmt -d cmd) <(printf "")
|
||||
- diff -au <(gofmt -d pkg) <(printf "")
|
||||
- make test GOFLAGS="-timeout 15m -race -v"
|
||||
- make coverage
|
||||
|
||||
|
||||
+13
-2
@@ -1,5 +1,7 @@
|
||||
FROM alpine:3.5
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
@@ -9,11 +11,20 @@ WORKDIR /go/src/github.com/minio/
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps git go musl-dev && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-04-29T00:40:27Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-04-29T00-40-27Z -X github.com/minio/minio/cmd.CommitID=eb50175ad911d496bf583a599de89547f0c9be89" && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
|
||||
rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
ENTRYPOINT ["minio"]
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
CMD ["minio"]
|
||||
|
||||
+13
-2
@@ -1,5 +1,7 @@
|
||||
FROM resin/aarch64-alpine:3.5
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
@@ -9,11 +11,20 @@ WORKDIR /go/src/github.com/minio/
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps git go musl-dev && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-03-16T21:50:32Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-03-16T21-50-32Z -X github.com/minio/minio/cmd.CommitID=5311eb22fd681a8cd4a46e2a872d46c2352c64e8" && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
|
||||
rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
ENTRYPOINT ["minio"]
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
CMD ["minio"]
|
||||
|
||||
+13
-2
@@ -1,5 +1,7 @@
|
||||
FROM resin/armhf-alpine:3.5
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
@@ -9,11 +11,20 @@ WORKDIR /go/src/github.com/minio/
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps git go musl-dev && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
go get -v -d github.com/minio/minio && \
|
||||
cd /go/src/github.com/minio/minio && \
|
||||
go install -v -ldflags "-X github.com/minio/minio/cmd.Version=2017-03-16T21:50:32Z -X github.com/minio/minio/cmd.ReleaseTag=RELEASE.2017-03-16T21-50-32Z -X github.com/minio/minio/cmd.CommitID=5311eb22fd681a8cd4a46e2a872d46c2352c64e8" && \
|
||||
go install -v -ldflags "$(go run buildscripts/gen-ldflags.go)" && \
|
||||
rm -rf /go/pkg /go/src /usr/local/go && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
ENTRYPOINT ["minio"]
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
CMD ["minio"]
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
FROM alpine:3.5
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps curl && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl https://dl.minio.io/server/minio/release/linux-amd64/minio > /usr/bin/minio && \
|
||||
chmod +x /usr/bin/minio && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
CMD ["minio"]
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
FROM resin/aarch64-alpine:3.5
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps curl && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl https://dl.minio.io/server/minio/release/linux-arm64/minio > /usr/bin/minio && \
|
||||
chmod +x /usr/bin/minio && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
CMD ["minio"]
|
||||
@@ -0,0 +1,22 @@
|
||||
FROM resin/armhf-alpine:3.5
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps curl && \
|
||||
echo 'hosts: files mdns4_minimal [NOTFOUND=return] dns mdns4' >> /etc/nsswitch.conf && \
|
||||
curl https://dl.minio.io/server/minio/release/linux-arm/minio > /usr/bin/minio && \
|
||||
chmod +x /usr/bin/minio && apk del .build-deps
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
COPY buildscripts/docker-entrypoint.sh /usr/bin/
|
||||
|
||||
RUN chmod +x /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/export"]
|
||||
|
||||
CMD ["minio"]
|
||||
@@ -1,7 +1,7 @@
|
||||
LDFLAGS := $(shell go run buildscripts/gen-ldflags.go)
|
||||
PWD := $(shell pwd)
|
||||
GOPATH := $(shell go env GOPATH)
|
||||
BUILD_LDFLAGS := '$(LDFLAGS) -s -w'
|
||||
BUILD_LDFLAGS := '$(LDFLAGS)'
|
||||
TAG := latest
|
||||
|
||||
HOST ?= $(shell uname)
|
||||
@@ -56,40 +56,41 @@ endif
|
||||
all: install
|
||||
|
||||
checks:
|
||||
@echo "Checking deps:"
|
||||
@echo "Check deps"
|
||||
@(env bash $(PWD)/buildscripts/checkdeps.sh)
|
||||
@echo "Checking project is in GOPATH"
|
||||
@(env bash $(PWD)/buildscripts/checkgopath.sh)
|
||||
|
||||
getdeps: checks
|
||||
@echo "Installing golint:" && go get -u github.com/golang/lint/golint
|
||||
@echo "Installing gocyclo:" && go get -u github.com/fzipp/gocyclo
|
||||
@echo "Installing deadcode:" && go get -u github.com/remyoudompheng/go-misc/deadcode
|
||||
@echo "Installing misspell:" && go get -u github.com/client9/misspell/cmd/misspell
|
||||
@echo "Installing ineffassign:" && go get -u github.com/gordonklaus/ineffassign
|
||||
@echo "Installing golint" && go get -u github.com/golang/lint/golint
|
||||
@echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo
|
||||
@echo "Installing deadcode" && go get -u github.com/remyoudompheng/go-misc/deadcode
|
||||
@echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell
|
||||
@echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign
|
||||
|
||||
verifiers: vet fmt lint cyclo spelling
|
||||
|
||||
vet:
|
||||
@echo "Running $@:"
|
||||
@go vet github.com/minio/minio/cmd/...
|
||||
@go vet github.com/minio/minio/pkg/...
|
||||
@echo "Running $@"
|
||||
@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult cmd
|
||||
@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult pkg
|
||||
|
||||
fmt:
|
||||
@echo "Running $@:"
|
||||
@gofmt -s -l cmd
|
||||
@gofmt -s -l pkg
|
||||
@echo "Running $@"
|
||||
@gofmt -d cmd
|
||||
@gofmt -d pkg
|
||||
|
||||
lint:
|
||||
@echo "Running $@:"
|
||||
@echo "Running $@"
|
||||
@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/cmd...
|
||||
@${GOPATH}/bin/golint -set_exit_status github.com/minio/minio/pkg...
|
||||
|
||||
ineffassign:
|
||||
@echo "Running $@:"
|
||||
@echo "Running $@"
|
||||
@${GOPATH}/bin/ineffassign .
|
||||
|
||||
cyclo:
|
||||
@echo "Running $@:"
|
||||
@echo "Running $@"
|
||||
@${GOPATH}/bin/gocyclo -over 100 cmd
|
||||
@${GOPATH}/bin/gocyclo -over 100 pkg
|
||||
|
||||
@@ -104,27 +105,30 @@ spelling:
|
||||
@${GOPATH}/bin/misspell -error `find docs/`
|
||||
|
||||
test: build
|
||||
@echo "Running all minio testing:"
|
||||
@echo "Running all minio testing"
|
||||
@go test $(GOFLAGS) .
|
||||
@go test $(GOFLAGS) github.com/minio/minio/cmd...
|
||||
@go test $(GOFLAGS) github.com/minio/minio/pkg...
|
||||
|
||||
coverage: build
|
||||
@echo "Running all coverage for minio:"
|
||||
@echo "Running all coverage for minio"
|
||||
@./buildscripts/go-coverage.sh
|
||||
|
||||
gomake-all: build
|
||||
@echo "Installing minio:"
|
||||
@echo "Installing minio at $(GOPATH)/bin/minio"
|
||||
@go build --ldflags $(BUILD_LDFLAGS) -o $(GOPATH)/bin/minio
|
||||
|
||||
pkg-add:
|
||||
${GOPATH}/bin/govendor add $(PKG)
|
||||
@echo "Adding new package $(PKG)"
|
||||
@${GOPATH}/bin/govendor add $(PKG)
|
||||
|
||||
pkg-update:
|
||||
${GOPATH}/bin/govendor update $(PKG)
|
||||
@echo "Updating new package $(PKG)"
|
||||
@${GOPATH}/bin/govendor update $(PKG)
|
||||
|
||||
pkg-remove:
|
||||
${GOPATH}/bin/govendor remove $(PKG)
|
||||
@echo "Remove new package $(PKG)"
|
||||
@${GOPATH}/bin/govendor remove $(PKG)
|
||||
|
||||
pkg-list:
|
||||
@$(GOPATH)/bin/govendor list
|
||||
@@ -138,7 +142,7 @@ experimental: verifiers
|
||||
@MINIO_RELEASE=EXPERIMENTAL ./buildscripts/build.sh
|
||||
|
||||
clean:
|
||||
@echo "Cleaning up all the generated files:"
|
||||
@echo "Cleaning up all the generated files"
|
||||
@find . -name '*.test' | xargs rm -fv
|
||||
@rm -rf build
|
||||
@rm -rf release
|
||||
|
||||
@@ -26,6 +26,20 @@ Install minio packages using [Homebrew](http://brew.sh/)
|
||||
brew install minio/stable/minio
|
||||
minio server ~/Photos
|
||||
```
|
||||
#### Note
|
||||
If you previously installed minio using `brew install minio` then uninstall minio as shown below
|
||||
|
||||
```
|
||||
brew uninstall minio
|
||||
```
|
||||
|
||||
Then re-install the latest minio using:
|
||||
|
||||
```
|
||||
brew install minio/stable/minio
|
||||
```
|
||||
|
||||
>`brew install minio` and `brew upgrade minio` will no longer install/upgrade the latest minio binaries on macOS. Upstream bugs in golang 1.8 broke Minio brew installer. Use the updated `minio/stable/minio` in your brew paths.
|
||||
|
||||
### Binary Download
|
||||
| Platform| Architecture | URL|
|
||||
@@ -40,11 +54,11 @@ chmod 755 minio
|
||||
### Binary Download
|
||||
| Platform| Architecture | URL|
|
||||
| ----------| -------- | ------|
|
||||
|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
|
||||
||32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio|
|
||||
||32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio|
|
||||
||64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio|
|
||||
||32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio|
|
||||
|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
|
||||
| |32-bit Intel|https://dl.minio.io/server/minio/release/linux-386/minio |
|
||||
| |32-bit ARM|https://dl.minio.io/server/minio/release/linux-arm/minio |
|
||||
| |64-bit ARM|https://dl.minio.io/server/minio/release/linux-arm64/minio |
|
||||
| |32-bit ARMv6|https://dl.minio.io/server/minio/release/linux-arm6vl/minio |
|
||||
```sh
|
||||
chmod +x minio
|
||||
./minio server ~/Photos
|
||||
@@ -54,32 +68,38 @@ chmod +x minio
|
||||
### Binary Download
|
||||
| Platform| Architecture | URL|
|
||||
| ----------| -------- | ------|
|
||||
|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
|
||||
||32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe|
|
||||
|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
|
||||
| |32-bit|https://dl.minio.io/server/minio/release/windows-386/minio.exe |
|
||||
```sh
|
||||
minio.exe server D:\Photos
|
||||
```
|
||||
|
||||
## FreeBSD
|
||||
### Port
|
||||
Install minio packages using [pkg](https://github.com/freebsd/pkg)
|
||||
|
||||
```sh
|
||||
pkg install minio
|
||||
sysrc minio_enable=yes
|
||||
sysrc minio_disks=/home/user/Photos
|
||||
service minio start
|
||||
```
|
||||
|
||||
### Binary Download
|
||||
| Platform| Architecture | URL|
|
||||
| ----------| -------- | ------|
|
||||
|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
|
||||
|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio |
|
||||
```sh
|
||||
chmod 755 minio
|
||||
./minio server ~/Photos
|
||||
```
|
||||
|
||||
You can run Minio on FreeBSD with FreeNAS storage-backend - see [here](https://docs.minio.io/docs/how-to-run-minio-in-freenas) for more details.
|
||||
|
||||
## Install from Source
|
||||
|
||||
Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
|
||||
|
||||
```sh
|
||||
|
||||
go get -u github.com/minio/minio
|
||||
|
||||
```
|
||||
|
||||
## Test using Minio Browser
|
||||
|
||||
+3
-3
@@ -36,9 +36,9 @@ test_script:
|
||||
# Unit tests
|
||||
- ps: Add-AppveyorTest "Unit Tests" -Outcome Running
|
||||
- mkdir build\coverage
|
||||
- go test -timeout 17m -race github.com/minio/minio/cmd...
|
||||
- go test -race github.com/minio/minio/pkg...
|
||||
- go test -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
|
||||
- go test -v -timeout 17m -race github.com/minio/minio/cmd...
|
||||
- go test -v -race github.com/minio/minio/pkg...
|
||||
- go test -v -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
|
||||
- ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
|
||||
|
||||
after_test:
|
||||
|
||||
+8
-7
@@ -1,8 +1,9 @@
|
||||
{
|
||||
"presets": [
|
||||
"es2015",
|
||||
"react"
|
||||
],
|
||||
|
||||
"plugins": ["transform-object-rest-spread"]
|
||||
}
|
||||
"presets": [
|
||||
"es2015",
|
||||
"react"
|
||||
],
|
||||
"plugins": [
|
||||
"transform-object-rest-spread"
|
||||
]
|
||||
}
|
||||
+1
-1
@@ -27,7 +27,7 @@ go get github.com/elazarl/go-bindata-assetfs/...
|
||||
yarn release
|
||||
```
|
||||
|
||||
This generates ui-assets.go in the current direcotry. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
|
||||
This generates ui-assets.go in the current directory. Now do `make` in the parent directory to build the minio binary with the newly generated ``ui-assets.go``
|
||||
|
||||
### Run Minio Browser with live reload
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
import 'babel-polyfill'
|
||||
|
||||
import './less/main.less'
|
||||
|
||||
@@ -44,7 +45,6 @@ import Web from './js/web'
|
||||
window.Web = Web
|
||||
|
||||
import storage from 'local-storage-fallback'
|
||||
|
||||
const store = applyMiddleware(thunkMiddleware)(createStore)(reducer)
|
||||
const Browse = connect(state => state)(_Browse)
|
||||
const Login = connect(state => state)(_Login)
|
||||
|
||||
@@ -26,6 +26,8 @@ export const SET_BUCKETS = 'SET_BUCKETS'
|
||||
export const ADD_BUCKET = 'ADD_BUCKET'
|
||||
export const SET_VISIBLE_BUCKETS = 'SET_VISIBLE_BUCKETS'
|
||||
export const SET_OBJECTS = 'SET_OBJECTS'
|
||||
export const APPEND_OBJECTS = 'APPEND_OBJECTS'
|
||||
export const RESET_OBJECTS = 'RESET_OBJECTS'
|
||||
export const SET_STORAGE_INFO = 'SET_STORAGE_INFO'
|
||||
export const SET_SERVER_INFO = 'SET_SERVER_INFO'
|
||||
export const SHOW_MAKEBUCKET_MODAL = 'SHOW_MAKEBUCKET_MODAL'
|
||||
@@ -240,15 +242,28 @@ export const setVisibleBuckets = visibleBuckets => {
|
||||
}
|
||||
}
|
||||
|
||||
export const setObjects = (objects, marker, istruncated) => {
|
||||
const appendObjects = (objects, marker, istruncated) => {
|
||||
return {
|
||||
type: SET_OBJECTS,
|
||||
type: APPEND_OBJECTS,
|
||||
objects,
|
||||
marker,
|
||||
istruncated
|
||||
}
|
||||
}
|
||||
|
||||
export const setObjects = (objects) => {
|
||||
return {
|
||||
type: SET_OBJECTS,
|
||||
objects,
|
||||
}
|
||||
}
|
||||
|
||||
export const resetObjects = () => {
|
||||
return {
|
||||
type: RESET_OBJECTS
|
||||
}
|
||||
}
|
||||
|
||||
export const setCurrentBucket = currentBucket => {
|
||||
return {
|
||||
type: SET_CURRENT_BUCKET,
|
||||
@@ -316,7 +331,7 @@ export const listObjects = () => {
|
||||
object.name = object.name.replace(`${currentPath}`, '');
|
||||
return object
|
||||
})
|
||||
dispatch(setObjects(objects, res.nextmarker, res.istruncated))
|
||||
dispatch(appendObjects(objects, res.nextmarker, res.istruncated))
|
||||
dispatch(setPrefixWritable(res.writable))
|
||||
dispatch(setLoadBucket(''))
|
||||
dispatch(setLoadPath(''))
|
||||
@@ -337,7 +352,7 @@ export const listObjects = () => {
|
||||
export const selectPrefix = prefix => {
|
||||
return (dispatch, getState) => {
|
||||
const {currentBucket, web} = getState()
|
||||
dispatch(setObjects([], "", false))
|
||||
dispatch(resetObjects())
|
||||
dispatch(setLoadPath(prefix))
|
||||
web.ListObjects({
|
||||
bucketName: currentBucket,
|
||||
@@ -352,7 +367,7 @@ export const selectPrefix = prefix => {
|
||||
object.name = object.name.replace(`${prefix}`, '');
|
||||
return object
|
||||
})
|
||||
dispatch(setObjects(
|
||||
dispatch(appendObjects(
|
||||
objects,
|
||||
res.nextmarker,
|
||||
res.istruncated
|
||||
@@ -417,6 +432,8 @@ export const setLoginError = () => {
|
||||
|
||||
export const downloadSelected = (url, req, xhr) => {
|
||||
return (dispatch) => {
|
||||
var anchor = document.createElement('a')
|
||||
document.body.appendChild(anchor);
|
||||
xhr.open('POST', url, true)
|
||||
xhr.responseType = 'blob'
|
||||
|
||||
@@ -424,10 +441,20 @@ export const downloadSelected = (url, req, xhr) => {
|
||||
if (this.status == 200) {
|
||||
dispatch(checkedObjectsReset())
|
||||
var blob = new Blob([this.response], {
|
||||
type: 'application/zip'
|
||||
type: 'octet/stream'
|
||||
})
|
||||
var blobUrl = window.URL.createObjectURL(blob);
|
||||
window.location = blobUrl
|
||||
var separator = req.prefix.length > 1 ? '-' : ''
|
||||
|
||||
anchor.href = blobUrl
|
||||
anchor.download = req.bucketName+separator+req.prefix.slice(0, -1)+'.zip';
|
||||
|
||||
|
||||
|
||||
|
||||
anchor.click()
|
||||
window.URL.revokeObjectURL(blobUrl)
|
||||
anchor.remove()
|
||||
}
|
||||
};
|
||||
xhr.send(JSON.stringify(req));
|
||||
|
||||
@@ -68,7 +68,7 @@ export default class Browse extends React.Component {
|
||||
memory: res.MinioMemory,
|
||||
platform: res.MinioPlatform,
|
||||
runtime: res.MinioRuntime,
|
||||
envVars: res.MinioEnvVars
|
||||
info: res.MinioGlobalInfo
|
||||
})
|
||||
dispatch(actions.setServerInfo(serverInfo))
|
||||
})
|
||||
@@ -227,7 +227,12 @@ export default class Browse extends React.Component {
|
||||
|
||||
removeObject() {
|
||||
const {web, dispatch, currentPath, currentBucket, deleteConfirmation, checkedObjects} = this.props
|
||||
let objects = checkedObjects.length > 0 ? checkedObjects : [deleteConfirmation.object]
|
||||
let objects = []
|
||||
if (checkedObjects.length > 0) {
|
||||
objects = checkedObjects.map(obj => `${currentPath}${obj}`)
|
||||
} else {
|
||||
objects = [deleteConfirmation.object]
|
||||
}
|
||||
|
||||
web.RemoveObject({
|
||||
bucketname: currentBucket,
|
||||
@@ -458,21 +463,25 @@ export default class Browse extends React.Component {
|
||||
}
|
||||
|
||||
if (web.LoggedIn()) {
|
||||
storageUsageDetails = <div className="feh-usage">
|
||||
<div className="fehu-chart">
|
||||
<div style={ { width: usedPercent } }></div>
|
||||
</div>
|
||||
<ul>
|
||||
<li>
|
||||
Used:
|
||||
{ humanize.filesize(total - free) }
|
||||
</li>
|
||||
<li className="pull-right">
|
||||
Free:
|
||||
{ humanize.filesize(total - used) }
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
if (!(used === 0 && free === 0)) {
|
||||
storageUsageDetails = <div className="feh-usage">
|
||||
<div className="fehu-chart">
|
||||
<div style={ { width: usedPercent } }></div>
|
||||
</div>
|
||||
<ul>
|
||||
<li>
|
||||
<span>Used: </span>
|
||||
{ humanize.filesize(total - free) }
|
||||
</li>
|
||||
<li className="pull-right">
|
||||
<span>Free: </span>
|
||||
{ humanize.filesize(total - used) }
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
let createButton = ''
|
||||
@@ -717,11 +726,11 @@ export default class Browse extends React.Component {
|
||||
</div>
|
||||
<div className="input-group" style={ { display: web.LoggedIn() ? 'block' : 'none' } }>
|
||||
<label>
|
||||
Expires in
|
||||
Expires in (Max 7 days)
|
||||
</label>
|
||||
<div className="set-expire">
|
||||
<div className="set-expire-item">
|
||||
<i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) }></i>
|
||||
<i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireDays', 1, shareObject.object) } />
|
||||
<div className="set-expire-title">
|
||||
Days
|
||||
</div>
|
||||
@@ -730,12 +739,14 @@ export default class Browse extends React.Component {
|
||||
type="number"
|
||||
min={ 0 }
|
||||
max={ 7 }
|
||||
defaultValue={ 5 } />
|
||||
defaultValue={ 5 }
|
||||
readOnly="readOnly"
|
||||
/>
|
||||
</div>
|
||||
<i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) }></i>
|
||||
<i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireDays', -1, shareObject.object) } />
|
||||
</div>
|
||||
<div className="set-expire-item">
|
||||
<i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) }></i>
|
||||
<i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireHours', 1, shareObject.object) } />
|
||||
<div className="set-expire-title">
|
||||
Hours
|
||||
</div>
|
||||
@@ -744,12 +755,14 @@ export default class Browse extends React.Component {
|
||||
type="number"
|
||||
min={ 0 }
|
||||
max={ 23 }
|
||||
defaultValue={ 0 } />
|
||||
defaultValue={ 0 }
|
||||
readOnly="readOnly"
|
||||
/>
|
||||
</div>
|
||||
<i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) }></i>
|
||||
<i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireHours', -1, shareObject.object) } />
|
||||
</div>
|
||||
<div className="set-expire-item">
|
||||
<i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) }></i>
|
||||
<i className="set-expire-increase" onClick={ this.handleExpireValue.bind(this, 'expireMins', 1, shareObject.object) } />
|
||||
<div className="set-expire-title">
|
||||
Minutes
|
||||
</div>
|
||||
@@ -758,11 +771,13 @@ export default class Browse extends React.Component {
|
||||
type="number"
|
||||
min={ 0 }
|
||||
max={ 59 }
|
||||
defaultValue={ 0 } />
|
||||
defaultValue={ 0 }
|
||||
readOnly="readOnly"
|
||||
/>
|
||||
</div>
|
||||
<i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) }></i>
|
||||
<i className="set-expire-decrease" onClick={ this.handleExpireValue.bind(this, 'expireMins', -1, shareObject.object) } />
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</ModalBody>
|
||||
<div className="modal-footer">
|
||||
|
||||
@@ -89,10 +89,6 @@ export default class Login extends React.Component {
|
||||
{ alertBox }
|
||||
<div className="l-wrap">
|
||||
<form onSubmit={ this.handleSubmit.bind(this) }>
|
||||
<input name="fixBrowser"
|
||||
autoComplete="username"
|
||||
type="text"
|
||||
style={ { display: 'none' } } />
|
||||
<InputGroup className="ig-dark"
|
||||
label="Access Key"
|
||||
id="accessKey"
|
||||
@@ -102,7 +98,6 @@ export default class Login extends React.Component {
|
||||
required="required"
|
||||
autoComplete="username">
|
||||
</InputGroup>
|
||||
<input type="text" autoComplete="new-password" style={ { display: 'none' } } />
|
||||
<InputGroup className="ig-dark"
|
||||
label="Secret Key"
|
||||
id="secretKey"
|
||||
|
||||
@@ -7,6 +7,7 @@ import * as actions from '../actions'
|
||||
class PolicyInput extends Component {
|
||||
componentDidMount() {
|
||||
const {web, dispatch} = this.props
|
||||
this.prefix.focus()
|
||||
web.ListAllBucketPolicies({
|
||||
bucketName: this.props.currentBucket
|
||||
}).then(res => {
|
||||
@@ -27,8 +28,23 @@ class PolicyInput extends Component {
|
||||
|
||||
handlePolicySubmit(e) {
|
||||
e.preventDefault()
|
||||
const {web, dispatch} = this.props
|
||||
const {web, dispatch, currentBucket} = this.props
|
||||
|
||||
let prefix = currentBucket + '/' + this.prefix.value
|
||||
let policy = this.policy.value
|
||||
|
||||
if (!prefix.endsWith('*')) prefix = prefix + '*'
|
||||
|
||||
let prefixAlreadyExists = this.props.policies.some(elem => prefix === elem.prefix)
|
||||
|
||||
if (prefixAlreadyExists) {
|
||||
dispatch(actions.showAlert({
|
||||
type: 'danger',
|
||||
message: "Policy for this prefix already exists."
|
||||
}))
|
||||
return
|
||||
}
|
||||
|
||||
web.SetBucketPolicy({
|
||||
bucketName: this.props.currentBucket,
|
||||
prefix: this.prefix.value,
|
||||
@@ -36,8 +52,7 @@ class PolicyInput extends Component {
|
||||
})
|
||||
.then(() => {
|
||||
dispatch(actions.setPolicies([{
|
||||
policy: this.policy.value,
|
||||
prefix: this.prefix.value + '*',
|
||||
policy, prefix
|
||||
}, ...this.props.policies]))
|
||||
this.prefix.value = ''
|
||||
})
|
||||
|
||||
@@ -34,23 +34,12 @@ class SettingsModal extends React.Component {
|
||||
|
||||
let accessKeyEnv = ''
|
||||
let secretKeyEnv = ''
|
||||
// Check environment variables first. They may or may not have been
|
||||
// loaded already; they load in Browse#componentDidMount.
|
||||
if (serverInfo.envVars) {
|
||||
serverInfo.envVars.forEach(envVar => {
|
||||
let keyVal = envVar.split('=')
|
||||
if (keyVal[0] == 'MINIO_ACCESS_KEY') {
|
||||
accessKeyEnv = keyVal[1]
|
||||
} else if (keyVal[0] == 'MINIO_SECRET_KEY') {
|
||||
secretKeyEnv = keyVal[1]
|
||||
}
|
||||
})
|
||||
}
|
||||
if (accessKeyEnv != '' || secretKeyEnv != '') {
|
||||
// Check environment variables first.
|
||||
if (serverInfo.info.isEnvCreds) {
|
||||
dispatch(actions.setSettings({
|
||||
accessKey: accessKeyEnv,
|
||||
secretKey: secretKeyEnv,
|
||||
keysReadOnly: true
|
||||
accessKey: 'xxxxxxxxx',
|
||||
secretKey: 'xxxxxxxxx',
|
||||
keysReadOnly: true
|
||||
}))
|
||||
} else {
|
||||
web.GetAuth()
|
||||
|
||||
@@ -77,16 +77,18 @@ export default (state = {
|
||||
case actions.SET_CURRENT_BUCKET:
|
||||
newState.currentBucket = action.currentBucket
|
||||
break
|
||||
case actions.APPEND_OBJECTS:
|
||||
newState.objects = [...newState.objects, ...action.objects]
|
||||
newState.marker = action.marker
|
||||
newState.istruncated = action.istruncated
|
||||
break
|
||||
case actions.SET_OBJECTS:
|
||||
if (!action.objects.length) {
|
||||
newState.objects = []
|
||||
newState.marker = ""
|
||||
newState.istruncated = action.istruncated
|
||||
} else {
|
||||
newState.objects = [...action.objects]
|
||||
newState.marker = action.marker
|
||||
newState.istruncated = action.istruncated
|
||||
}
|
||||
newState.objects = [...action.objects]
|
||||
break
|
||||
case actions.RESET_OBJECTS:
|
||||
newState.objects = []
|
||||
newState.marker = ""
|
||||
newState.istruncated = false
|
||||
break
|
||||
case actions.SET_CURRENT_PATH:
|
||||
newState.currentPath = action.currentPath
|
||||
|
||||
@@ -81,7 +81,7 @@ select.form-control {
|
||||
width: 100%;
|
||||
height: 40px;
|
||||
border: 0;
|
||||
background: transparent;
|
||||
background: transparent !important;
|
||||
text-align: center;
|
||||
position: relative;
|
||||
z-index: 1;
|
||||
@@ -114,8 +114,8 @@ select.form-control {
|
||||
|
||||
.ig-dark {
|
||||
.ig-text {
|
||||
color: @white;
|
||||
border-color: rgba(255,255,255,0.1);
|
||||
color: @white !important;
|
||||
border-color: rgba(255,255,255,0.1) !important;
|
||||
}
|
||||
|
||||
.ig-helpers {
|
||||
@@ -183,6 +183,17 @@ select.form-control {
|
||||
.set-expire {
|
||||
border: 1px solid @input-border;
|
||||
margin: 35px 0 30px;
|
||||
position: relative;
|
||||
|
||||
&:before {
|
||||
content: '';
|
||||
position: absolute;
|
||||
left: 0;
|
||||
top: 0;
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
z-index: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.set-expire-item {
|
||||
@@ -191,6 +202,7 @@ select.form-control {
|
||||
display: table-cell;
|
||||
width: 1%;
|
||||
text-align: center;
|
||||
.user-select(none);
|
||||
|
||||
&:not(:last-child) {
|
||||
border-right: 1px solid @input-border;
|
||||
@@ -209,6 +221,7 @@ select.form-control {
|
||||
left: -8px;
|
||||
|
||||
input {
|
||||
.user-select(none);
|
||||
font-size: 20px;
|
||||
text-align: center;
|
||||
position: relative;
|
||||
|
||||
+50
-46
File diff suppressed because one or more lines are too long
@@ -22,7 +22,6 @@ var purify = require("purifycss-webpack-plugin")
|
||||
var exports = {
|
||||
context: __dirname,
|
||||
entry: [
|
||||
"babel-polyfill",
|
||||
path.resolve(__dirname, 'app/index.js')
|
||||
],
|
||||
output: {
|
||||
@@ -101,7 +100,6 @@ var exports = {
|
||||
|
||||
if (process.env.NODE_ENV === 'dev') {
|
||||
exports.entry = [
|
||||
"babel-polyfill",
|
||||
'webpack/hot/dev-server',
|
||||
'webpack-dev-server/client?http://localhost:8080',
|
||||
path.resolve(__dirname, 'app/index.js')
|
||||
|
||||
+158
-153
@@ -1,10 +1,12 @@
|
||||
# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
|
||||
# yarn lockfile v1
|
||||
|
||||
|
||||
abbrev@1:
|
||||
version "1.0.9"
|
||||
resolved "https://registry.yarnpkg.com/abbrev/-/abbrev-1.0.9.tgz#91b4792588a7738c25f35dd6f63752a2f8776135"
|
||||
|
||||
accepts@~1.3.3, accepts@1.3.3:
|
||||
accepts@1.3.3, accepts@~1.3.3:
|
||||
version "1.3.3"
|
||||
resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.3.tgz#c3ca7434938648c3e0d9c1e328dd68b622c284ca"
|
||||
dependencies:
|
||||
@@ -591,6 +593,14 @@ babel-polyfill@^6.22.0:
|
||||
core-js "^2.4.0"
|
||||
regenerator-runtime "^0.10.0"
|
||||
|
||||
babel-polyfill@^6.23.0:
|
||||
version "6.23.0"
|
||||
resolved "https://registry.yarnpkg.com/babel-polyfill/-/babel-polyfill-6.23.0.tgz#8364ca62df8eafb830499f699177466c3b03499d"
|
||||
dependencies:
|
||||
babel-runtime "^6.22.0"
|
||||
core-js "^2.4.0"
|
||||
regenerator-runtime "^0.10.0"
|
||||
|
||||
babel-preset-es2015@^6.14.0:
|
||||
version "6.22.0"
|
||||
resolved "https://registry.yarnpkg.com/babel-preset-es2015/-/babel-preset-es2015-6.22.0.tgz#af5a98ecb35eb8af764ad8a5a05eb36dc4386835"
|
||||
@@ -690,14 +700,14 @@ babel-types@^6.19.0, babel-types@^6.22.0:
|
||||
lodash "^4.2.0"
|
||||
to-fast-properties "^1.0.1"
|
||||
|
||||
babylon@^6.11.0, babylon@^6.15.0, babylon@^6.8.0:
|
||||
version "6.15.0"
|
||||
resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.15.0.tgz#ba65cfa1a80e1759b0e89fb562e27dccae70348e"
|
||||
|
||||
babylon@6.14.1:
|
||||
version "6.14.1"
|
||||
resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.14.1.tgz#956275fab72753ad9b3435d7afe58f8bf0a29815"
|
||||
|
||||
babylon@^6.11.0, babylon@^6.15.0, babylon@^6.8.0:
|
||||
version "6.15.0"
|
||||
resolved "https://registry.yarnpkg.com/babylon/-/babylon-6.15.0.tgz#ba65cfa1a80e1759b0e89fb562e27dccae70348e"
|
||||
|
||||
backo2@1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/backo2/-/backo2-1.0.2.tgz#31ab1ac8b129363463e35b3ebb69f4dfcfba7947"
|
||||
@@ -722,7 +732,7 @@ base64id@1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/base64id/-/base64id-1.0.0.tgz#47688cb99bb6804f0e06d3e763b1c32e57d8e6b6"
|
||||
|
||||
batch@^0.5.3, batch@0.5.3:
|
||||
batch@0.5.3, batch@^0.5.3:
|
||||
version "0.5.3"
|
||||
resolved "https://registry.yarnpkg.com/batch/-/batch-0.5.3.tgz#3f3414f380321743bfc1042f9a83ff1d5824d464"
|
||||
|
||||
@@ -936,7 +946,7 @@ classnames@^2.1.5, classnames@^2.2.3, classnames@^2.2.5:
|
||||
version "2.2.5"
|
||||
resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.2.5.tgz#fb3801d453467649ef3603c7d61a02bd129bde6d"
|
||||
|
||||
clean-css@^3.2.10, clean-css@3.4.x:
|
||||
clean-css@3.4.x, clean-css@^3.2.10:
|
||||
version "3.4.24"
|
||||
resolved "https://registry.yarnpkg.com/clean-css/-/clean-css-3.4.24.tgz#89f5a5e9da37ae02394fe049a41388abbe72c3b5"
|
||||
dependencies:
|
||||
@@ -1015,12 +1025,6 @@ combined-stream@^1.0.5, combined-stream@~1.0.5:
|
||||
dependencies:
|
||||
delayed-stream "~1.0.0"
|
||||
|
||||
commander@^2.2.0, commander@^2.5.0, commander@^2.8.1, commander@^2.9.0, commander@2.9.x:
|
||||
version "2.9.0"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"
|
||||
dependencies:
|
||||
graceful-readlink ">= 1.0.0"
|
||||
|
||||
commander@0.6.1:
|
||||
version "0.6.1"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-0.6.1.tgz#fa68a14f6a945d54dbbe50d8cdb3320e9e3b1a06"
|
||||
@@ -1035,6 +1039,12 @@ commander@2.8.x:
|
||||
dependencies:
|
||||
graceful-readlink ">= 1.0.0"
|
||||
|
||||
commander@2.9.x, commander@^2.2.0, commander@^2.5.0, commander@^2.8.1, commander@^2.9.0:
|
||||
version "2.9.0"
|
||||
resolved "https://registry.yarnpkg.com/commander/-/commander-2.9.0.tgz#9c99094176e12240cb22d6c5146098400fe0f7d4"
|
||||
dependencies:
|
||||
graceful-readlink ">= 1.0.0"
|
||||
|
||||
commondir@^1.0.1:
|
||||
version "1.0.1"
|
||||
resolved "https://registry.yarnpkg.com/commondir/-/commondir-1.0.1.tgz#ddd800da0c66127393cca5950ea968a3aaf1253b"
|
||||
@@ -1057,14 +1067,14 @@ component-bind@1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/component-bind/-/component-bind-1.0.0.tgz#00c608ab7dcd93897c0009651b1d3a8e1e73bbd1"
|
||||
|
||||
component-emitter@~1.2.0, component-emitter@1.2.1:
|
||||
version "1.2.1"
|
||||
resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6"
|
||||
|
||||
component-emitter@1.1.2:
|
||||
version "1.1.2"
|
||||
resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.1.2.tgz#296594f2753daa63996d2af08d15a95116c9aec3"
|
||||
|
||||
component-emitter@1.2.1, component-emitter@~1.2.0:
|
||||
version "1.2.1"
|
||||
resolved "https://registry.yarnpkg.com/component-emitter/-/component-emitter-1.2.1.tgz#137918d6d78283f7df7a6b7c5a63e140e69425e6"
|
||||
|
||||
component-inherit@0.0.3:
|
||||
version "0.0.3"
|
||||
resolved "https://registry.yarnpkg.com/component-inherit/-/component-inherit-0.0.3.tgz#645fc4adf58b72b649d5cae65135619db26ff143"
|
||||
@@ -1144,7 +1154,7 @@ cookie-signature@1.0.6:
|
||||
version "1.0.6"
|
||||
resolved "https://registry.yarnpkg.com/cookie-signature/-/cookie-signature-1.0.6.tgz#e303a882b342cc3ee8ca513a79999734dab3ae2c"
|
||||
|
||||
cookie@^0.3.1, cookie@0.3.1:
|
||||
cookie@0.3.1, cookie@^0.3.1:
|
||||
version "0.3.1"
|
||||
resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.3.1.tgz#e7e0a1f9ef43b4c8ba925c5c5a96e806d16873bb"
|
||||
|
||||
@@ -1313,17 +1323,13 @@ date-now@^0.1.4:
|
||||
version "0.1.4"
|
||||
resolved "https://registry.yarnpkg.com/date-now/-/date-now-0.1.4.tgz#eaf439fd4d4848ad74e5cc7dbef200672b9e345b"
|
||||
|
||||
debug@^0.7.4:
|
||||
version "0.7.4"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-0.7.4.tgz#06e1ea8082c2cb14e39806e22e2f6f757f92af39"
|
||||
|
||||
debug@^2.1.1, debug@^2.1.3, debug@^2.2.0, debug@2, debug@2.6.0:
|
||||
debug@2, debug@2.6.0, debug@^2.1.1, debug@^2.1.3, debug@^2.2.0:
|
||||
version "2.6.0"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.0.tgz#bc596bcabe7617f11d9fa15361eded5608b8499b"
|
||||
dependencies:
|
||||
ms "0.7.2"
|
||||
|
||||
debug@~2.2.0, debug@2.2.0:
|
||||
debug@2.2.0, debug@~2.2.0:
|
||||
version "2.2.0"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-2.2.0.tgz#f87057e995b1a1f6ae6a4960664137bc56f039da"
|
||||
dependencies:
|
||||
@@ -1335,6 +1341,10 @@ debug@2.3.3:
|
||||
dependencies:
|
||||
ms "0.7.2"
|
||||
|
||||
debug@^0.7.4:
|
||||
version "0.7.4"
|
||||
resolved "https://registry.yarnpkg.com/debug/-/debug-0.7.4.tgz#06e1ea8082c2cb14e39806e22e2f6f757f92af39"
|
||||
|
||||
decamelize@^1.0.0, decamelize@^1.1.1, decamelize@^1.1.2:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290"
|
||||
@@ -1391,7 +1401,7 @@ di@^0.0.1:
|
||||
version "0.0.1"
|
||||
resolved "https://registry.yarnpkg.com/di/-/di-0.0.1.tgz#806649326ceaa7caa3306d75d985ea2748ba913c"
|
||||
|
||||
diff@^1.3.2, diff@1.4.0:
|
||||
diff@1.4.0, diff@^1.3.2:
|
||||
version "1.4.0"
|
||||
resolved "https://registry.yarnpkg.com/diff/-/diff-1.4.0.tgz#7f28d2eb9ee7b15a97efd89ce63dcfdaa3ccbabf"
|
||||
|
||||
@@ -1440,14 +1450,14 @@ domain-browser@^1.1.1:
|
||||
version "1.1.7"
|
||||
resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.1.7.tgz#867aa4b093faa05f1de08c06f4d7b21fdf8698bc"
|
||||
|
||||
domelementtype@~1.1.1:
|
||||
version "1.1.3"
|
||||
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.1.3.tgz#bd28773e2642881aec51544924299c5cd822185b"
|
||||
|
||||
domelementtype@1:
|
||||
version "1.3.0"
|
||||
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.3.0.tgz#b17aed82e8ab59e52dd9c19b1756e0fc187204c2"
|
||||
|
||||
domelementtype@~1.1.1:
|
||||
version "1.1.3"
|
||||
resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-1.1.3.tgz#bd28773e2642881aec51544924299c5cd822185b"
|
||||
|
||||
domhandler@2.1:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-2.1.0.tgz#d2646f5e57f6c3bab11cf6cb05d3c0acf7412594"
|
||||
@@ -1587,7 +1597,7 @@ escape-html@~1.0.3:
|
||||
version "1.0.3"
|
||||
resolved "https://registry.yarnpkg.com/escape-html/-/escape-html-1.0.3.tgz#0258eae4d3d0c0974de1c169188ef0051d1d1988"
|
||||
|
||||
escape-string-regexp@^1.0.2, escape-string-regexp@1.0.2:
|
||||
escape-string-regexp@1.0.2, escape-string-regexp@^1.0.2:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-1.0.2.tgz#4dbc2fe674e71949caf3fb2695ce7f2dc1d9a8d1"
|
||||
|
||||
@@ -1754,7 +1764,7 @@ express@^4.13.3:
|
||||
utils-merge "1.0.0"
|
||||
vary "~1.1.0"
|
||||
|
||||
extend@^3.0.0, extend@~3.0.0, extend@3.0.0:
|
||||
extend@3.0.0, extend@^3.0.0, extend@~3.0.0:
|
||||
version "3.0.0"
|
||||
resolved "https://registry.yarnpkg.com/extend/-/extend-3.0.0.tgz#5a474353b9f3353ddd8176dfd37b91c83a46f1d4"
|
||||
|
||||
@@ -1859,14 +1869,6 @@ forever-agent@~0.6.1:
|
||||
version "0.6.1"
|
||||
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
|
||||
|
||||
form-data@~2.1.1:
|
||||
version "2.1.2"
|
||||
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.2.tgz#89c3534008b97eada4cbb157d58f6f5df025eae4"
|
||||
dependencies:
|
||||
asynckit "^0.4.0"
|
||||
combined-stream "^1.0.5"
|
||||
mime-types "^2.1.12"
|
||||
|
||||
form-data@1.0.0-rc3:
|
||||
version "1.0.0-rc3"
|
||||
resolved "https://registry.yarnpkg.com/form-data/-/form-data-1.0.0-rc3.tgz#d35bc62e7fbc2937ae78f948aaa0d38d90607577"
|
||||
@@ -1875,6 +1877,14 @@ form-data@1.0.0-rc3:
|
||||
combined-stream "^1.0.5"
|
||||
mime-types "^2.1.3"
|
||||
|
||||
form-data@~2.1.1:
|
||||
version "2.1.2"
|
||||
resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.1.2.tgz#89c3534008b97eada4cbb157d58f6f5df025eae4"
|
||||
dependencies:
|
||||
asynckit "^0.4.0"
|
||||
combined-stream "^1.0.5"
|
||||
mime-types "^2.1.12"
|
||||
|
||||
formidable@~1.0.14:
|
||||
version "1.0.17"
|
||||
resolved "https://registry.yarnpkg.com/formidable/-/formidable-1.0.17.tgz#ef5491490f9433b705faa77249c99029ae348559"
|
||||
@@ -1978,6 +1988,13 @@ glob-parent@^2.0.0:
|
||||
dependencies:
|
||||
is-glob "^2.0.0"
|
||||
|
||||
glob@3.2.11:
|
||||
version "3.2.11"
|
||||
resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d"
|
||||
dependencies:
|
||||
inherits "2"
|
||||
minimatch "0.3"
|
||||
|
||||
glob@^5.0.15:
|
||||
version "5.0.15"
|
||||
resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1"
|
||||
@@ -2009,13 +2026,6 @@ glob@^7.0.0, glob@^7.0.5:
|
||||
once "^1.3.0"
|
||||
path-is-absolute "^1.0.0"
|
||||
|
||||
glob@3.2.11:
|
||||
version "3.2.11"
|
||||
resolved "https://registry.yarnpkg.com/glob/-/glob-3.2.11.tgz#4a973f635b9190f715d10987d5c00fd2815ebe3d"
|
||||
dependencies:
|
||||
inherits "2"
|
||||
minimatch "0.3"
|
||||
|
||||
globals@^9.0.0:
|
||||
version "9.14.0"
|
||||
resolved "https://registry.yarnpkg.com/globals/-/globals-9.14.0.tgz#8859936af0038741263053b39d0e76ca241e4034"
|
||||
@@ -2194,7 +2204,7 @@ humanize@0.0.9:
|
||||
version "0.0.9"
|
||||
resolved "https://registry.yarnpkg.com/humanize/-/humanize-0.0.9.tgz#1994ffaecdfe9c441ed2bdac7452b7bb4c9e41a4"
|
||||
|
||||
iconv-lite@^0.4.5, iconv-lite@0.4.15:
|
||||
iconv-lite@0.4.15, iconv-lite@^0.4.5:
|
||||
version "0.4.15"
|
||||
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.15.tgz#fe265a218ac6a57cfe854927e9d04c19825eddeb"
|
||||
|
||||
@@ -2225,7 +2235,7 @@ inflight@^1.0.4:
|
||||
once "^1.3.0"
|
||||
wrappy "1"
|
||||
|
||||
inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1, inherits@2, inherits@2.0.3:
|
||||
inherits@2, inherits@2.0.3, inherits@^2.0.1, inherits@~2.0.0, inherits@~2.0.1:
|
||||
version "2.0.3"
|
||||
resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de"
|
||||
|
||||
@@ -2418,14 +2428,14 @@ is-typedarray@~1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
|
||||
|
||||
isarray@^1.0.0, isarray@~1.0.0, isarray@1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
|
||||
|
||||
isarray@0.0.1:
|
||||
version "0.0.1"
|
||||
resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf"
|
||||
|
||||
isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
|
||||
|
||||
isbinaryfile@^3.0.0:
|
||||
version "3.0.2"
|
||||
resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-3.0.2.tgz#4a3e974ec0cba9004d3fc6cde7209ea69368a621"
|
||||
@@ -2505,7 +2515,7 @@ json-stringify-safe@~5.0.1:
|
||||
version "5.0.1"
|
||||
resolved "https://registry.yarnpkg.com/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz#1296a2d58fd45f19a0f6ce01d65701e2c735b6eb"
|
||||
|
||||
json3@^3.3.2, json3@3.3.2:
|
||||
json3@3.3.2, json3@^3.3.2:
|
||||
version "3.3.2"
|
||||
resolved "https://registry.yarnpkg.com/json3/-/json3-3.3.2.tgz#3c0434743df93e2f5c42aee7b19bcb483575f4e1"
|
||||
|
||||
@@ -2633,7 +2643,7 @@ less@^2.7.1:
|
||||
request "^2.72.0"
|
||||
source-map "^0.5.3"
|
||||
|
||||
loader-utils@^0.2.11, loader-utils@^0.2.16, loader-utils@^0.2.5, loader-utils@^0.2.7, loader-utils@~0.2.2, loader-utils@0.2.x:
|
||||
loader-utils@0.2.x, loader-utils@^0.2.11, loader-utils@^0.2.16, loader-utils@^0.2.5, loader-utils@^0.2.7, loader-utils@~0.2.2:
|
||||
version "0.2.16"
|
||||
resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-0.2.16.tgz#f08632066ed8282835dff88dfb52704765adee6d"
|
||||
dependencies:
|
||||
@@ -2726,20 +2736,16 @@ lower-case@^1.1.1:
|
||||
version "1.1.3"
|
||||
resolved "https://registry.yarnpkg.com/lower-case/-/lower-case-1.1.3.tgz#c92393d976793eee5ba4edb583cf8eae35bd9bfb"
|
||||
|
||||
lru-cache@2, lru-cache@2.2.x:
|
||||
version "2.2.4"
|
||||
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.2.4.tgz#6c658619becf14031d0d0b594b16042ce4dc063d"
|
||||
|
||||
lru-cache@^3.2.0:
|
||||
version "3.2.0"
|
||||
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-3.2.0.tgz#71789b3b7f5399bec8565dda38aa30d2a097efee"
|
||||
dependencies:
|
||||
pseudomap "^1.0.1"
|
||||
|
||||
lru-cache@2:
|
||||
version "2.7.3"
|
||||
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.7.3.tgz#6d4524e8b955f95d4f5b58851ce21dd72fb4e952"
|
||||
|
||||
lru-cache@2.2.x:
|
||||
version "2.2.4"
|
||||
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-2.2.4.tgz#6c658619becf14031d0d0b594b16042ce4dc063d"
|
||||
|
||||
macaddress@^0.2.8:
|
||||
version "0.2.8"
|
||||
resolved "https://registry.yarnpkg.com/macaddress/-/macaddress-0.2.8.tgz#5904dc537c39ec6dbefeae902327135fa8511f12"
|
||||
@@ -2798,7 +2804,7 @@ micromatch@^2.1.5, micromatch@^2.3.11:
|
||||
parse-glob "^3.0.4"
|
||||
regex-cache "^0.4.2"
|
||||
|
||||
mime-db@^1.25.0, "mime-db@>= 1.24.0 < 2", mime-db@~1.26.0:
|
||||
"mime-db@>= 1.24.0 < 2", mime-db@^1.25.0, mime-db@~1.26.0:
|
||||
version "1.26.0"
|
||||
resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.26.0.tgz#eaffcd0e4fc6935cf8134da246e2e6c35305adff"
|
||||
|
||||
@@ -2808,19 +2814,13 @@ mime-types@^2.1.12, mime-types@^2.1.13, mime-types@^2.1.3, mime-types@~2.1.11, m
|
||||
dependencies:
|
||||
mime-db "~1.26.0"
|
||||
|
||||
mime@^1.2.11, mime@^1.3.4, mime@1.3.4:
|
||||
version "1.3.4"
|
||||
resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53"
|
||||
|
||||
mime@1.2.x:
|
||||
version "1.2.11"
|
||||
resolved "https://registry.yarnpkg.com/mime/-/mime-1.2.11.tgz#58203eed86e3a5ef17aed2b7d9ebd47f0a60dd10"
|
||||
|
||||
minimatch@^3.0.0, minimatch@^3.0.2, "minimatch@2 || 3":
|
||||
version "3.0.3"
|
||||
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774"
|
||||
dependencies:
|
||||
brace-expansion "^1.0.0"
|
||||
mime@1.3.4, mime@^1.2.11, mime@^1.3.4:
|
||||
version "1.3.4"
|
||||
resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.4.tgz#115f9e3b6b3daf2959983cb38f149a2d40eb5d53"
|
||||
|
||||
minimatch@0.3:
|
||||
version "0.3.0"
|
||||
@@ -2829,24 +2829,30 @@ minimatch@0.3:
|
||||
lru-cache "2"
|
||||
sigmund "~1.0.0"
|
||||
|
||||
"minimatch@2 || 3", minimatch@^3.0.0, minimatch@^3.0.2:
|
||||
version "3.0.3"
|
||||
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774"
|
||||
dependencies:
|
||||
brace-expansion "^1.0.0"
|
||||
|
||||
minimist@0.0.8, minimist@~0.0.1:
|
||||
version "0.0.8"
|
||||
resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
|
||||
|
||||
minimist@^1.1.1, minimist@^1.2.0:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.0.tgz#a35008b20f41383eec1fb914f4cd5df79a264284"
|
||||
|
||||
minimist@~0.0.1, minimist@0.0.8:
|
||||
version "0.0.8"
|
||||
resolved "https://registry.yarnpkg.com/minimist/-/minimist-0.0.8.tgz#857fcabfc3397d2625b8228262e86aa7a011b05d"
|
||||
mkdirp@0.3.0:
|
||||
version "0.3.0"
|
||||
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e"
|
||||
|
||||
mkdirp@^0.5.0, mkdirp@^0.5.1, "mkdirp@>=0.5 0", mkdirp@~0.5.0, mkdirp@~0.5.1, mkdirp@0.5.1:
|
||||
mkdirp@0.5.1, "mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.0, mkdirp@~0.5.1:
|
||||
version "0.5.1"
|
||||
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.1.tgz#30057438eac6cf7f8c4767f38648d6697d75c903"
|
||||
dependencies:
|
||||
minimist "0.0.8"
|
||||
|
||||
mkdirp@0.3.0:
|
||||
version "0.3.0"
|
||||
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.3.0.tgz#1bbf5ab1ba827af23575143490426455f481fe1e"
|
||||
|
||||
mocha@^2.5.3:
|
||||
version "2.5.3"
|
||||
resolved "https://registry.yarnpkg.com/mocha/-/mocha-2.5.3.tgz#161be5bdeb496771eb9b35745050b622b5aefc58"
|
||||
@@ -2866,14 +2872,14 @@ moment@^2.15.1:
|
||||
version "2.17.1"
|
||||
resolved "https://registry.yarnpkg.com/moment/-/moment-2.17.1.tgz#fed9506063f36b10f066c8b59a144d7faebe1d82"
|
||||
|
||||
mout@^0.11.0:
|
||||
version "0.11.1"
|
||||
resolved "https://registry.yarnpkg.com/mout/-/mout-0.11.1.tgz#ba3611df5f0e5b1ffbfd01166b8f02d1f5fa2b99"
|
||||
|
||||
"mout@>=0.9 <2.0":
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/mout/-/mout-1.0.0.tgz#9bdf1d4af57d66d47cb353a6335a3281098e1501"
|
||||
|
||||
mout@^0.11.0:
|
||||
version "0.11.1"
|
||||
resolved "https://registry.yarnpkg.com/mout/-/mout-0.11.1.tgz#ba3611df5f0e5b1ffbfd01166b8f02d1f5fa2b99"
|
||||
|
||||
ms@0.7.1:
|
||||
version "0.7.1"
|
||||
resolved "https://registry.yarnpkg.com/ms/-/ms-0.7.1.tgz#9cd13c03adbff25b65effde7ce864ee952017098"
|
||||
@@ -3028,6 +3034,10 @@ oauth-sign@~0.8.1:
|
||||
version "0.8.2"
|
||||
resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.8.2.tgz#46a6ab7f0aead8deae9ec0565780b7d4efeb9d43"
|
||||
|
||||
object-assign@4.1.0:
|
||||
version "4.1.0"
|
||||
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0"
|
||||
|
||||
object-assign@^2.0.0:
|
||||
version "2.1.1"
|
||||
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-2.1.1.tgz#43c36e5d569ff8e4816c4efa8be02d26967c18aa"
|
||||
@@ -3036,10 +3046,6 @@ object-assign@^4.0.1, object-assign@^4.1.0:
|
||||
version "4.1.1"
|
||||
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
|
||||
|
||||
object-assign@4.1.0:
|
||||
version "4.1.0"
|
||||
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.0.tgz#7a3b3d0e98063d43f4c03f2e8ae6cd51a86883a0"
|
||||
|
||||
object-component@0.0.3:
|
||||
version "0.0.3"
|
||||
resolved "https://registry.yarnpkg.com/object-component/-/object-component-0.0.3.tgz#f0c69aa50efc95b866c186f400a33769cb2f1291"
|
||||
@@ -3518,14 +3524,14 @@ pseudomap@^1.0.1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3"
|
||||
|
||||
punycode@^1.2.4, punycode@^1.4.1:
|
||||
version "1.4.1"
|
||||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
|
||||
|
||||
punycode@1.3.2:
|
||||
version "1.3.2"
|
||||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d"
|
||||
|
||||
punycode@^1.2.4, punycode@^1.4.1:
|
||||
version "1.4.1"
|
||||
resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e"
|
||||
|
||||
purify-css@^1.1.1:
|
||||
version "1.1.9"
|
||||
resolved "https://registry.yarnpkg.com/purify-css/-/purify-css-1.1.9.tgz#46c9acd8940f3076c0c346c027e286f996168357"
|
||||
@@ -3548,10 +3554,6 @@ q@^1.1.2:
|
||||
version "1.4.1"
|
||||
resolved "https://registry.yarnpkg.com/q/-/q-1.4.1.tgz#55705bcd93c5f3673530c2c2cbc0c2b3addc286e"
|
||||
|
||||
qs@~6.3.0:
|
||||
version "6.3.0"
|
||||
resolved "https://registry.yarnpkg.com/qs/-/qs-6.3.0.tgz#f403b264f23bc01228c74131b407f18d5ea5d442"
|
||||
|
||||
qs@2.3.3:
|
||||
version "2.3.3"
|
||||
resolved "https://registry.yarnpkg.com/qs/-/qs-2.3.3.tgz#e9e85adbe75da0bbe4c8e0476a086290f863b404"
|
||||
@@ -3564,6 +3566,10 @@ qs@6.2.1:
|
||||
version "6.2.1"
|
||||
resolved "https://registry.yarnpkg.com/qs/-/qs-6.2.1.tgz#ce03c5ff0935bc1d9d69a9f14cbd18e568d67625"
|
||||
|
||||
qs@~6.3.0:
|
||||
version "6.3.0"
|
||||
resolved "https://registry.yarnpkg.com/qs/-/qs-6.3.0.tgz#f403b264f23bc01228c74131b407f18d5ea5d442"
|
||||
|
||||
query-string@^3.0.0:
|
||||
version "3.0.3"
|
||||
resolved "https://registry.yarnpkg.com/query-string/-/query-string-3.0.3.tgz#ae2e14b4d05071d4e9b9eb4873c35b0dcd42e638"
|
||||
@@ -3720,6 +3726,24 @@ react@^0.14.8:
|
||||
envify "^3.0.0"
|
||||
fbjs "^0.6.1"
|
||||
|
||||
readable-stream@1.0, readable-stream@~1.0.2:
|
||||
version "1.0.34"
|
||||
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c"
|
||||
dependencies:
|
||||
core-util-is "~1.0.0"
|
||||
inherits "~2.0.1"
|
||||
isarray "0.0.1"
|
||||
string_decoder "~0.10.x"
|
||||
|
||||
readable-stream@1.0.27-1:
|
||||
version "1.0.27-1"
|
||||
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.27-1.tgz#6b67983c20357cefd07f0165001a16d710d91078"
|
||||
dependencies:
|
||||
core-util-is "~1.0.0"
|
||||
inherits "~2.0.1"
|
||||
isarray "0.0.1"
|
||||
string_decoder "~0.10.x"
|
||||
|
||||
"readable-stream@^2.0.0 || ^1.1.13", readable-stream@^2.0.1, readable-stream@^2.0.2, readable-stream@^2.0.5, readable-stream@^2.1.0:
|
||||
version "2.2.2"
|
||||
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.2.2.tgz#a9e6fec3c7dda85f8bb1b3ba7028604556fc825e"
|
||||
@@ -3732,15 +3756,6 @@ react@^0.14.8:
|
||||
string_decoder "~0.10.x"
|
||||
util-deprecate "~1.0.1"
|
||||
|
||||
readable-stream@~1.0.2, readable-stream@1.0:
|
||||
version "1.0.34"
|
||||
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.34.tgz#125820e34bc842d2f2aaafafe4c2916ee32c157c"
|
||||
dependencies:
|
||||
core-util-is "~1.0.0"
|
||||
inherits "~2.0.1"
|
||||
isarray "0.0.1"
|
||||
string_decoder "~0.10.x"
|
||||
|
||||
readable-stream@~2.1.4:
|
||||
version "2.1.5"
|
||||
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.1.5.tgz#66fa8b720e1438b364681f2ad1a63c618448c9d0"
|
||||
@@ -3753,15 +3768,6 @@ readable-stream@~2.1.4:
|
||||
string_decoder "~0.10.x"
|
||||
util-deprecate "~1.0.1"
|
||||
|
||||
readable-stream@1.0.27-1:
|
||||
version "1.0.27-1"
|
||||
resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-1.0.27-1.tgz#6b67983c20357cefd07f0165001a16d710d91078"
|
||||
dependencies:
|
||||
core-util-is "~1.0.0"
|
||||
inherits "~2.0.1"
|
||||
isarray "0.0.1"
|
||||
string_decoder "~0.10.x"
|
||||
|
||||
readdirp@^2.0.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/readdirp/-/readdirp-2.1.0.tgz#4ed0ad060df3073300c48440373f72d1cc642d78"
|
||||
@@ -3948,7 +3954,7 @@ right-align@^0.1.1:
|
||||
dependencies:
|
||||
align-text "^0.1.1"
|
||||
|
||||
rimraf@^2.3.3, rimraf@~2.5.1, rimraf@~2.5.4, rimraf@2:
|
||||
rimraf@2, rimraf@^2.3.3, rimraf@~2.5.1, rimraf@~2.5.4:
|
||||
version "2.5.4"
|
||||
resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.5.4.tgz#96800093cbf1a0c86bd95b4625467535c29dfa04"
|
||||
dependencies:
|
||||
@@ -3990,7 +3996,7 @@ rocambole-whitespace@^1.0.0:
|
||||
repeat-string "^1.5.0"
|
||||
rocambole-token "^1.2.1"
|
||||
|
||||
rocambole@^0.7.0, "rocambole@>=0.7 <2.0":
|
||||
"rocambole@>=0.7 <2.0", rocambole@^0.7.0:
|
||||
version "0.7.0"
|
||||
resolved "https://registry.yarnpkg.com/rocambole/-/rocambole-0.7.0.tgz#f6c79505517dc42b6fb840842b8b953b0f968585"
|
||||
dependencies:
|
||||
@@ -4208,28 +4214,28 @@ source-map-url@~0.3.0:
|
||||
version "0.3.0"
|
||||
resolved "https://registry.yarnpkg.com/source-map-url/-/source-map-url-0.3.0.tgz#7ecaf13b57bcd09da8a40c5d269db33799d4aaf9"
|
||||
|
||||
source-map@0.1.34:
|
||||
version "0.1.34"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.1.34.tgz#a7cfe89aec7b1682c3b198d0acfb47d7d090566b"
|
||||
dependencies:
|
||||
amdefine ">=0.0.4"
|
||||
|
||||
source-map@0.4.x, source-map@^0.4.2, source-map@~0.4.1:
|
||||
version "0.4.4"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b"
|
||||
dependencies:
|
||||
amdefine ">=0.0.4"
|
||||
|
||||
source-map@^0.1.38, source-map@^0.1.41:
|
||||
version "0.1.43"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.1.43.tgz#c24bc146ca517c1471f5dacbe2571b2b7f9e3346"
|
||||
dependencies:
|
||||
amdefine ">=0.0.4"
|
||||
|
||||
source-map@^0.4.2, source-map@~0.4.1, source-map@0.4.x:
|
||||
version "0.4.4"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b"
|
||||
dependencies:
|
||||
amdefine ">=0.0.4"
|
||||
|
||||
source-map@^0.5.0, source-map@^0.5.3, source-map@^0.5.6, source-map@~0.5.0, source-map@~0.5.1, source-map@~0.5.3:
|
||||
version "0.5.6"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.6.tgz#75ce38f52bf0733c5a7f0c118d81334a2bb5f412"
|
||||
|
||||
source-map@0.1.34:
|
||||
version "0.1.34"
|
||||
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.1.34.tgz#a7cfe89aec7b1682c3b198d0acfb47d7d090566b"
|
||||
dependencies:
|
||||
amdefine ">=0.0.4"
|
||||
|
||||
sprintf-js@~1.0.2:
|
||||
version "1.0.3"
|
||||
resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
|
||||
@@ -4282,10 +4288,6 @@ strict-uri-encode@^1.0.0:
|
||||
version "1.1.0"
|
||||
resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-1.1.0.tgz#279b225df1d582b1f54e65addd4352e18faa0713"
|
||||
|
||||
string_decoder@^0.10.25, string_decoder@~0.10.x:
|
||||
version "0.10.31"
|
||||
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
|
||||
|
||||
string-width@^1.0.1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/string-width/-/string-width-1.0.2.tgz#118bdf5b8cdc51a2a7e70d211e07e2b0b9b107d3"
|
||||
@@ -4294,6 +4296,10 @@ string-width@^1.0.1:
|
||||
is-fullwidth-code-point "^1.0.0"
|
||||
strip-ansi "^3.0.0"
|
||||
|
||||
string_decoder@^0.10.25, string_decoder@~0.10.x:
|
||||
version "0.10.31"
|
||||
resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94"
|
||||
|
||||
stringstream@~0.0.4:
|
||||
version "0.0.5"
|
||||
resolved "https://registry.yarnpkg.com/stringstream/-/stringstream-0.0.5.tgz#4e484cd4de5a0bbbee18e46307710a8a81621878"
|
||||
@@ -4338,6 +4344,10 @@ superagent@^1.8.4:
|
||||
readable-stream "1.0.27-1"
|
||||
reduce-component "1.0.1"
|
||||
|
||||
supports-color@1.2.0:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-1.2.0.tgz#ff1ed1e61169d06b3cf2d588e188b18d8847e17e"
|
||||
|
||||
supports-color@^0.2.0:
|
||||
version "0.2.0"
|
||||
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-0.2.0.tgz#d92de2694eb3f67323973d7ae3d8b55b4c22190a"
|
||||
@@ -4356,10 +4366,6 @@ supports-color@^3.1.0, supports-color@^3.1.1, supports-color@^3.2.3:
|
||||
dependencies:
|
||||
has-flag "^1.0.0"
|
||||
|
||||
supports-color@1.2.0:
|
||||
version "1.2.0"
|
||||
resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-1.2.0.tgz#ff1ed1e61169d06b3cf2d588e188b18d8847e17e"
|
||||
|
||||
svgo@^0.7.0:
|
||||
version "0.7.1"
|
||||
resolved "https://registry.yarnpkg.com/svgo/-/svgo-0.7.1.tgz#287320fed972cb097e72c2bb1685f96fe08f8034"
|
||||
@@ -4488,7 +4494,7 @@ ua-parser-js@^0.7.9:
|
||||
version "0.7.12"
|
||||
resolved "https://registry.yarnpkg.com/ua-parser-js/-/ua-parser-js-0.7.12.tgz#04c81a99bdd5dc52263ea29d24c6bf8d4818a4bb"
|
||||
|
||||
uglify-js@~2.7.3, uglify-js@2.7.x:
|
||||
uglify-js@2.7.x, uglify-js@~2.7.3:
|
||||
version "2.7.5"
|
||||
resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.7.5.tgz#4612c0c7baaee2ba7c487de4904ae122079f2ca8"
|
||||
dependencies:
|
||||
@@ -4538,7 +4544,7 @@ uniqs@^2.0.0:
|
||||
version "2.0.0"
|
||||
resolved "https://registry.yarnpkg.com/uniqs/-/uniqs-2.0.0.tgz#ffede4b36b25290696e6e165d4a59edb998e6b02"
|
||||
|
||||
unpipe@~1.0.0, unpipe@1.0.0:
|
||||
unpipe@1.0.0, unpipe@~1.0.0:
|
||||
version "1.0.0"
|
||||
resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec"
|
||||
|
||||
@@ -4557,16 +4563,16 @@ url-loader@^0.5.7:
|
||||
loader-utils "0.2.x"
|
||||
mime "1.2.x"
|
||||
|
||||
url-parse@^1.1.1:
|
||||
version "1.1.7"
|
||||
resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.1.7.tgz#025cff999653a459ab34232147d89514cc87d74a"
|
||||
url-parse@1.0.x:
|
||||
version "1.0.5"
|
||||
resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.0.5.tgz#0854860422afdcfefeb6c965c662d4800169927b"
|
||||
dependencies:
|
||||
querystringify "0.0.x"
|
||||
requires-port "1.0.x"
|
||||
|
||||
url-parse@1.0.x:
|
||||
version "1.0.5"
|
||||
resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.0.5.tgz#0854860422afdcfefeb6c965c662d4800169927b"
|
||||
url-parse@^1.1.1:
|
||||
version "1.1.7"
|
||||
resolved "https://registry.yarnpkg.com/url-parse/-/url-parse-1.1.7.tgz#025cff999653a459ab34232147d89514cc87d74a"
|
||||
dependencies:
|
||||
querystringify "0.0.x"
|
||||
requires-port "1.0.x"
|
||||
@@ -4598,7 +4604,7 @@ util-deprecate@~1.0.1:
|
||||
version "1.0.2"
|
||||
resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
|
||||
|
||||
util@^0.10.3, util@0.10.3:
|
||||
util@0.10.3, util@^0.10.3:
|
||||
version "0.10.3"
|
||||
resolved "https://registry.yarnpkg.com/util/-/util-0.10.3.tgz#7afb1afe50805246489e3db7fe0ed379336ac0f9"
|
||||
dependencies:
|
||||
@@ -4765,22 +4771,22 @@ wide-align@^1.1.0:
|
||||
dependencies:
|
||||
string-width "^1.0.1"
|
||||
|
||||
window-size@^0.1.4:
|
||||
version "0.1.4"
|
||||
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.4.tgz#f8e1aa1ee5a53ec5bf151ffa09742a6ad7697876"
|
||||
|
||||
window-size@0.1.0:
|
||||
version "0.1.0"
|
||||
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d"
|
||||
|
||||
wordwrap@~0.0.2:
|
||||
version "0.0.3"
|
||||
resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107"
|
||||
window-size@^0.1.4:
|
||||
version "0.1.4"
|
||||
resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.4.tgz#f8e1aa1ee5a53ec5bf151ffa09742a6ad7697876"
|
||||
|
||||
wordwrap@0.0.2:
|
||||
version "0.0.2"
|
||||
resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.2.tgz#b79669bb42ecb409f83d583cad52ca17eaa1643f"
|
||||
|
||||
wordwrap@~0.0.2:
|
||||
version "0.0.3"
|
||||
resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.3.tgz#a3d5da6cd5c0bc0008d37234bbaf1bed63059107"
|
||||
|
||||
wrap-ansi@^2.0.0:
|
||||
version "2.1.0"
|
||||
resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-2.1.0.tgz#d8fc3d284dd05794fe84973caecdd1cf824fdd85"
|
||||
@@ -4847,4 +4853,3 @@ yargs@~3.10.0:
|
||||
yeast@0.1.2:
|
||||
version "0.1.2"
|
||||
resolved "https://registry.yarnpkg.com/yeast/-/yeast-0.1.2.tgz#008e06d8094320c372dbc2f8ed76a0ca6c8ac419"
|
||||
|
||||
|
||||
+27
-48
@@ -5,7 +5,7 @@ _init() {
|
||||
LDFLAGS=$(go run buildscripts/gen-ldflags.go)
|
||||
|
||||
# Extract release tag
|
||||
release_tag=$(echo $LDFLAGS | awk {'print $4'} | cut -f2 -d=)
|
||||
release_tag=$(echo $LDFLAGS | awk {'print $6'} | cut -f2 -d=)
|
||||
|
||||
# Verify release tag.
|
||||
if [ -z "$release_tag" ]; then
|
||||
@@ -28,6 +28,7 @@ _init() {
|
||||
## System binaries
|
||||
CP=`which cp`
|
||||
SHASUM=`which shasum`
|
||||
SHA256SUM="${SHASUM} -a 256"
|
||||
SED=`which sed`
|
||||
}
|
||||
|
||||
@@ -43,58 +44,36 @@ go_build() {
|
||||
release_bin="$release_str/$os-$arch/$(basename $package).$release_tag"
|
||||
# Release binary downloadable name
|
||||
release_real_bin="$release_str/$os-$arch/$(basename $package)"
|
||||
# Release shasum name
|
||||
release_shasum="$release_str/$os-$arch/$(basename $package).shasum"
|
||||
|
||||
# Release sha1sum name
|
||||
release_shasum="$release_str/$os-$arch/$(basename $package).${release_tag}.shasum"
|
||||
# Release sha1sum default
|
||||
release_shasum_default="$release_str/$os-$arch/$(basename $package).shasum"
|
||||
|
||||
# Release sha256sum name
|
||||
release_sha256sum="$release_str/$os-$arch/$(basename $package).${release_tag}.sha256sum"
|
||||
# Release sha256sum default
|
||||
release_sha256sum_default="$release_str/$os-$arch/$(basename $package).sha256sum"
|
||||
|
||||
# Go build to build the binary.
|
||||
if [ "${arch}" == "arm" ]; then
|
||||
# Release binary downloadable name
|
||||
release_real_bin_6="$release_str/$os-${arch}6vl/$(basename $package)"
|
||||
CGO_ENABLED=0 GOOS=$os GOARCH=$arch go build --ldflags "${LDFLAGS}" -o $release_bin
|
||||
|
||||
release_bin_6="$release_str/$os-${arch}6vl/$(basename $package).$release_tag"
|
||||
## Support building for ARM6vl
|
||||
GOARM=6 GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin_6
|
||||
|
||||
## Copy
|
||||
$CP -p $release_bin_6 $release_real_bin_6
|
||||
|
||||
# Release shasum name
|
||||
release_shasum_6="$release_str/$os-${arch}6vl/$(basename $package).shasum"
|
||||
|
||||
# Calculate shasum
|
||||
shasum_str=$(${SHASUM} ${release_bin_6})
|
||||
echo ${shasum_str} | $SED "s/$release_str\/$os-${arch}6vl\///g" > $release_shasum_6
|
||||
|
||||
# Release binary downloadable name
|
||||
release_real_bin_7="$release_str/$os-$arch/$(basename $package)"
|
||||
|
||||
release_bin_7="$release_str/$os-$arch/$(basename $package).$release_tag"
|
||||
## Support building for ARM7vl
|
||||
GOARM=7 GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin_7
|
||||
|
||||
## Copy
|
||||
$CP -p $release_bin_7 $release_real_bin_7
|
||||
|
||||
# Release shasum name
|
||||
release_shasum_7="$release_str/$os-$arch/$(basename $package).shasum"
|
||||
|
||||
# Calculate shasum
|
||||
shasum_str=$(${SHASUM} ${release_bin_7})
|
||||
echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum_7
|
||||
# Create copy
|
||||
if [ $os == "windows" ]; then
|
||||
$CP -p $release_bin ${release_real_bin}.exe
|
||||
else
|
||||
GOOS=$os GOARCH=$arch go build --ldflags "-s -w ${LDFLAGS}" -o $release_bin
|
||||
|
||||
# Create copy
|
||||
if [ $os == "windows" ]; then
|
||||
$CP -p $release_bin ${release_real_bin}.exe
|
||||
else
|
||||
$CP -p $release_bin $release_real_bin
|
||||
fi
|
||||
|
||||
# Calculate shasum
|
||||
shasum_str=$(${SHASUM} ${release_bin})
|
||||
echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
|
||||
$CP -p $release_bin $release_real_bin
|
||||
fi
|
||||
|
||||
# Calculate sha1sum
|
||||
shasum_str=$(${SHASUM} ${release_bin})
|
||||
echo ${shasum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_shasum
|
||||
$CP -p $release_shasum $release_shasum_default
|
||||
|
||||
# Calculate sha256sum
|
||||
sha256sum_str=$(${SHA256SUM} ${release_bin})
|
||||
echo ${sha256sum_str} | $SED "s/$release_str\/$os-$arch\///g" > $release_sha256sum
|
||||
$CP -p $release_sha256sum $release_sha256sum_default
|
||||
}
|
||||
|
||||
main() {
|
||||
|
||||
@@ -25,6 +25,11 @@ _init() {
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
case "${KNAME}" in
|
||||
SunOS )
|
||||
ARCH=$(isainfo -k)
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
## FIXME:
|
||||
@@ -97,7 +102,7 @@ assert_is_supported_arch() {
|
||||
|
||||
assert_is_supported_os() {
|
||||
case "${KNAME}" in
|
||||
Linux | FreeBSD | OpenBSD | NetBSD | DragonFly )
|
||||
Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS )
|
||||
return
|
||||
;;
|
||||
Darwin )
|
||||
|
||||
@@ -16,8 +16,6 @@
|
||||
#
|
||||
|
||||
main() {
|
||||
echo "Checking project is in GOPATH:"
|
||||
|
||||
IFS=':' read -r -a paths <<< "$GOPATH"
|
||||
for path in "${paths[@]}"; do
|
||||
minio_path="$path/src/github.com/minio/minio"
|
||||
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/bin/sh
|
||||
#
|
||||
# Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
# If command starts with an option, prepend minio.
|
||||
if [ "${1}" != "minio" ]; then
|
||||
if [ -n "${1}" ]; then
|
||||
set -- minio "$@"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Wait for all the hosts to come online and have
|
||||
# their DNS entries populated properly.
|
||||
docker_wait_hosts() {
|
||||
hosts="$@"
|
||||
num_hosts=0
|
||||
# Count number of hosts in arguments.
|
||||
for host in $hosts; do
|
||||
[ $(echo "$host" | grep -E "^http") ] || continue
|
||||
num_hosts=$((num_hosts+1))
|
||||
done
|
||||
if [ $num_hosts -gt 0 ]; then
|
||||
echo -n "Waiting for all hosts to resolve..."
|
||||
while true; do
|
||||
x=0
|
||||
for host in $hosts; do
|
||||
[ $(echo "$host" | grep -E "^http") ] || continue
|
||||
# Extract the domain.
|
||||
host=$(echo $host | sed -e 's/^http[s]\?:\/\/\([^\/]\+\).*/\1/')
|
||||
echo -n .
|
||||
val=$(ping -c 1 $host 2>/dev/null)
|
||||
if [ $? != 0 ]; then
|
||||
echo "Failed to lookup $host"
|
||||
continue
|
||||
fi
|
||||
x=$((x+1))
|
||||
done
|
||||
# Provided hosts same as successful hosts, should break out.
|
||||
test $x -eq $num_hosts && break
|
||||
echo "Failed to resolve hosts.. retrying after 1 second."
|
||||
sleep 1
|
||||
done
|
||||
echo "All hosts are resolving proceeding to initialize Minio."
|
||||
fi
|
||||
}
|
||||
|
||||
## Look for docker secrets in default documented location.
|
||||
docker_secrets_env() {
|
||||
local MINIO_ACCESS_KEY_FILE="/run/secrets/access_key"
|
||||
local MINIO_SECRET_KEY_FILE="/run/secrets/secret_key"
|
||||
|
||||
if [ -f $MINIO_ACCESS_KEY_FILE -a -f $MINIO_SECRET_KEY_FILE ]; then
|
||||
if [ -f $MINIO_ACCESS_KEY_FILE ]; then
|
||||
export MINIO_ACCESS_KEY="$(cat "$MINIO_ACCESS_KEY_FILE")"
|
||||
fi
|
||||
if [ -f $MINIO_SECRET_KEY_FILE ]; then
|
||||
export MINIO_SECRET_KEY="$(cat "$MINIO_SECRET_KEY_FILE")"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
## Set access env from secrets if necessary.
|
||||
docker_secrets_env
|
||||
|
||||
## Wait for all the hosts to come online.
|
||||
docker_wait_hosts "$@"
|
||||
|
||||
exec "$@"
|
||||
@@ -27,8 +27,8 @@ import (
|
||||
)
|
||||
|
||||
func genLDFlags(version string) string {
|
||||
var ldflagsStr string
|
||||
ldflagsStr = "-X github.com/minio/minio/cmd.Version=" + version
|
||||
ldflagsStr := "-s -w"
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.Version=" + version
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag(version)
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID()
|
||||
ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12]
|
||||
|
||||
+183
-47
@@ -24,7 +24,9 @@ import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -49,6 +51,7 @@ const (
|
||||
mgmtDryRun mgmtQueryKey = "dry-run"
|
||||
mgmtUploadIDMarker mgmtQueryKey = "upload-id-marker"
|
||||
mgmtMaxUploads mgmtQueryKey = "max-uploads"
|
||||
mgmtUploadID mgmtQueryKey = "upload-id"
|
||||
)
|
||||
|
||||
// ServerVersion - server version
|
||||
@@ -205,14 +208,45 @@ type ServerConnStats struct {
|
||||
Throughput uint64 `json:"throughput,omitempty"`
|
||||
}
|
||||
|
||||
// ServerInfo holds the information that will be returned by ServerInfo API
|
||||
type ServerInfo struct {
|
||||
// ServerHTTPMethodStats holds total number of HTTP operations from/to the server,
|
||||
// including the average duration the call was spent.
|
||||
type ServerHTTPMethodStats struct {
|
||||
Count uint64 `json:"count"`
|
||||
AvgDuration string `json:"avgDuration"`
|
||||
}
|
||||
|
||||
// ServerHTTPStats holds all type of http operations performed to/from the server
|
||||
// including their average execution time.
|
||||
type ServerHTTPStats struct {
|
||||
TotalHEADStats ServerHTTPMethodStats `json:"totalHEADs"`
|
||||
SuccessHEADStats ServerHTTPMethodStats `json:"successHEADs"`
|
||||
TotalGETStats ServerHTTPMethodStats `json:"totalGETs"`
|
||||
SuccessGETStats ServerHTTPMethodStats `json:"successGETs"`
|
||||
TotalPUTStats ServerHTTPMethodStats `json:"totalPUTs"`
|
||||
SuccessPUTStats ServerHTTPMethodStats `json:"successPUTs"`
|
||||
TotalPOSTStats ServerHTTPMethodStats `json:"totalPOSTs"`
|
||||
SuccessPOSTStats ServerHTTPMethodStats `json:"successPOSTs"`
|
||||
TotalDELETEStats ServerHTTPMethodStats `json:"totalDELETEs"`
|
||||
SuccessDELETEStats ServerHTTPMethodStats `json:"successDELETEs"`
|
||||
}
|
||||
|
||||
// ServerInfoData holds storage, connections and other
|
||||
// information of a given server.
|
||||
type ServerInfoData struct {
|
||||
StorageInfo StorageInfo `json:"storage"`
|
||||
ConnStats ServerConnStats `json:"network"`
|
||||
HTTPStats ServerHTTPStats `json:"http"`
|
||||
Properties ServerProperties `json:"server"`
|
||||
}
|
||||
|
||||
// ServerInfoHandler - GET /?server-info
|
||||
// ServerInfo holds server information result of one node
|
||||
type ServerInfo struct {
|
||||
Error string `json:"error"`
|
||||
Addr string `json:"addr"`
|
||||
Data *ServerInfoData `json:"data"`
|
||||
}
|
||||
|
||||
// ServerInfoHandler - GET /?info
|
||||
// ----------
|
||||
// Get server information
|
||||
func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -223,58 +257,43 @@ func (adminAPI adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
|
||||
// Build storage info
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
storage := objLayer.StorageInfo()
|
||||
// Web service response
|
||||
reply := make([]ServerInfo, len(globalAdminPeers))
|
||||
|
||||
// Build list of enabled ARNs queues
|
||||
var arns []string
|
||||
for queueArn := range globalEventNotifier.GetAllExternalTargets() {
|
||||
arns = append(arns, queueArn)
|
||||
var wg sync.WaitGroup
|
||||
|
||||
// Gather server information for all nodes
|
||||
for i, p := range globalAdminPeers {
|
||||
wg.Add(1)
|
||||
|
||||
// Gather information from a peer in a goroutine
|
||||
go func(idx int, peer adminPeer) {
|
||||
defer wg.Done()
|
||||
|
||||
// Initialize server info at index
|
||||
reply[idx] = ServerInfo{Addr: peer.addr}
|
||||
|
||||
serverInfoData, err := peer.cmdRunner.ServerInfoData()
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to get server info from %s.", peer.addr)
|
||||
reply[idx].Error = err.Error()
|
||||
return
|
||||
}
|
||||
|
||||
reply[idx].Data = &serverInfoData
|
||||
}(i, p)
|
||||
}
|
||||
|
||||
// Fetch uptimes from all peers. This may fail to due to lack
|
||||
// of read-quorum availability.
|
||||
uptime, err := getPeerUptimes(globalAdminPeers)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
errorIf(err, "Unable to get uptime from majority of servers.")
|
||||
return
|
||||
}
|
||||
|
||||
// Build server properties information
|
||||
properties := ServerProperties{
|
||||
Version: Version,
|
||||
CommitID: CommitID,
|
||||
Region: serverConfig.GetRegion(),
|
||||
SQSARN: arns,
|
||||
Uptime: uptime,
|
||||
}
|
||||
|
||||
// Build network info
|
||||
connStats := ServerConnStats{
|
||||
TotalInputBytes: globalConnStats.getTotalInputBytes(),
|
||||
TotalOutputBytes: globalConnStats.getTotalOutputBytes(),
|
||||
}
|
||||
|
||||
// Build the whole returned information
|
||||
info := ServerInfo{
|
||||
StorageInfo: storage,
|
||||
ConnStats: connStats,
|
||||
Properties: properties,
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Marshal API response
|
||||
jsonBytes, err := json.Marshal(info)
|
||||
jsonBytes, err := json.Marshal(reply)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
errorIf(err, "Failed to marshal storage info into json.")
|
||||
return
|
||||
}
|
||||
|
||||
// Reply with storage information (across nodes in a
|
||||
// distributed setup) as json.
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
@@ -602,6 +621,42 @@ func isDryRun(qval url.Values) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// healResult - represents result of a heal operation like
|
||||
// heal-object, heal-upload.
|
||||
type healResult struct {
|
||||
State healState `json:"state"`
|
||||
}
|
||||
|
||||
// healState - different states of heal operation
|
||||
type healState int
|
||||
|
||||
const (
|
||||
// healNone - none of the disks healed
|
||||
healNone healState = iota
|
||||
// healPartial - some disks were healed, others were offline
|
||||
healPartial
|
||||
// healOK - all disks were healed
|
||||
healOK
|
||||
)
|
||||
|
||||
// newHealResult - returns healResult given number of disks healed and
|
||||
// number of disks offline
|
||||
func newHealResult(numHealedDisks, numOfflineDisks int) healResult {
|
||||
var state healState
|
||||
switch {
|
||||
case numHealedDisks == 0:
|
||||
state = healNone
|
||||
|
||||
case numOfflineDisks > 0:
|
||||
state = healPartial
|
||||
|
||||
default:
|
||||
state = healOK
|
||||
}
|
||||
|
||||
return healResult{State: state}
|
||||
}
|
||||
|
||||
// HealObjectHandler - POST /?heal&bucket=mybucket&object=myobject&dry-run
|
||||
// - x-minio-operation = object
|
||||
// - bucket and object are both mandatory query parameters
|
||||
@@ -644,14 +699,95 @@ func (adminAPI adminAPIHandlers) HealObjectHandler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
|
||||
err := objLayer.HealObject(bucket, object)
|
||||
numOfflineDisks, numHealedDisks, err := objLayer.HealObject(bucket, object)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Return 200 on success.
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
}
|
||||
|
||||
// HealUploadHandler - POST /?heal&bucket=mybucket&object=myobject&upload-id=myuploadID&dry-run
|
||||
// - x-minio-operation = upload
|
||||
// - bucket, object and upload-id are mandatory query parameters
|
||||
// Heal a given upload, if present.
|
||||
func (adminAPI adminAPIHandlers) HealUploadHandler(w http.ResponseWriter, r *http.Request) {
|
||||
// Get object layer instance.
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
adminAPIErr := checkRequestAuthType(r, "", "", "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponse(w, adminAPIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := r.URL.Query()
|
||||
bucket := vars.Get(string(mgmtBucket))
|
||||
object := vars.Get(string(mgmtObject))
|
||||
uploadID := vars.Get(string(mgmtUploadID))
|
||||
uploadObj := path.Join(bucket, object, uploadID)
|
||||
|
||||
// Validate bucket and object names as supplied via query
|
||||
// parameters.
|
||||
if err := checkBucketAndObjectNames(bucket, object); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate the bucket and object w.r.t backend representation
|
||||
// of an upload.
|
||||
if err := checkBucketAndObjectNames(minioMetaMultipartBucket,
|
||||
uploadObj); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Check if upload exists.
|
||||
if _, err := objLayer.GetObjectInfo(minioMetaMultipartBucket,
|
||||
uploadObj); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// if dry-run is set in query params then perform validations
|
||||
// and return success.
|
||||
if isDryRun(vars) {
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
return
|
||||
}
|
||||
|
||||
//We are able to use HealObject for healing an upload since an
|
||||
//ongoing upload has the same backend representation as an
|
||||
//object. The 'object' corresponding to a given bucket,
|
||||
//object and uploadID is
|
||||
//.minio.sys/multipart/bucket/object/uploadID.
|
||||
numOfflineDisks, numHealedDisks, err := objLayer.HealObject(minioMetaMultipartBucket, uploadObj)
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(newHealResult(numHealedDisks, numOfflineDisks))
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Return 200 on success.
|
||||
writeSuccessResponseJSON(w, jsonBytes)
|
||||
}
|
||||
|
||||
// HealFormatHandler - POST /?heal&dry-run
|
||||
|
||||
+309
-106
@@ -21,10 +21,12 @@ import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -154,14 +156,9 @@ func prepareAdminXLTestBed() (*adminXLTestBed, error) {
|
||||
}
|
||||
|
||||
// Initialize boot time
|
||||
globalBootTime = time.Now().UTC()
|
||||
globalBootTime = UTCNow()
|
||||
|
||||
// Set globalEndpoints for a single node XL setup.
|
||||
for _, xlDir := range xlDirs {
|
||||
globalEndpoints = append(globalEndpoints, &url.URL{
|
||||
Path: xlDir,
|
||||
})
|
||||
}
|
||||
globalEndpoints = mustGetNewEndpointList(xlDirs...)
|
||||
|
||||
// Set globalIsXL to indicate that the setup uses an erasure code backend.
|
||||
globalIsXL = true
|
||||
@@ -299,14 +296,8 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
|
||||
// Initialize admin peers to make admin RPC calls. Note: In a
|
||||
// single node setup, this degenerates to a simple function
|
||||
// call under the hood.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// Setting up a go routine to simulate ServerMux's
|
||||
// handleServiceSignals for stop and restart commands.
|
||||
@@ -365,14 +356,8 @@ func TestServiceSetCreds(t *testing.T) {
|
||||
// Initialize admin peers to make admin RPC calls. Note: In a
|
||||
// single node setup, this degenerates to a simple function
|
||||
// call under the hood.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
credentials := serverConfig.GetCredential()
|
||||
var body []byte
|
||||
@@ -453,14 +438,8 @@ func TestListLocksHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
@@ -528,11 +507,7 @@ func TestClearLocksHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
initGlobalAdminPeers(eps)
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
@@ -756,7 +731,7 @@ func TestListObjectsHealHandler(t *testing.T) {
|
||||
}
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
err = adminTestBed.objLayer.MakeBucket("mybucket")
|
||||
err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket - %v", err)
|
||||
}
|
||||
@@ -884,7 +859,7 @@ func TestHealBucketHandler(t *testing.T) {
|
||||
}
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
err = adminTestBed.objLayer.MakeBucket("mybucket")
|
||||
err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket - %v", err)
|
||||
}
|
||||
@@ -961,7 +936,7 @@ func TestHealObjectHandler(t *testing.T) {
|
||||
// Create an object myobject under bucket mybucket.
|
||||
bucketName := "mybucket"
|
||||
objName := "myobject"
|
||||
err = adminTestBed.objLayer.MakeBucket(bucketName)
|
||||
err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
|
||||
}
|
||||
@@ -1048,6 +1023,160 @@ func TestHealObjectHandler(t *testing.T) {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// buildAdminRequest - helper function to build an admin API request.
|
||||
func buildAdminRequest(queryVal url.Values, opHdr, method string,
|
||||
contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error) {
|
||||
req, err := newTestRequest(method, "/?"+queryVal.Encode(), contentLength, bodySeeker)
|
||||
if err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
|
||||
req.Header.Set(minioAdminOpHeader, opHdr)
|
||||
|
||||
cred := serverConfig.GetCredential()
|
||||
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
|
||||
if err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
|
||||
return req, nil
|
||||
}
|
||||
|
||||
// mkHealUploadQuery - helper to build HealUploadHandler query string.
|
||||
func mkHealUploadQuery(bucket, object, uploadID, dryRun string) url.Values {
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set(string(mgmtBucket), bucket)
|
||||
queryVal.Set(string(mgmtObject), object)
|
||||
queryVal.Set(string(mgmtUploadID), uploadID)
|
||||
queryVal.Set("heal", "")
|
||||
queryVal.Set(string(mgmtDryRun), dryRun)
|
||||
return queryVal
|
||||
}
|
||||
|
||||
// TestHealUploadHandler - test for HealUploadHandler.
|
||||
func TestHealUploadHandler(t *testing.T) {
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Create an object myobject under bucket mybucket.
|
||||
bucketName := "mybucket"
|
||||
objName := "myobject"
|
||||
err = adminTestBed.objLayer.MakeBucketWithLocation(bucketName, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket %s - %v", bucketName, err)
|
||||
}
|
||||
|
||||
// Create a new multipart upload.
|
||||
uploadID, err := adminTestBed.objLayer.NewMultipartUpload(bucketName, objName, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to create a new multipart upload %s/%s - %v",
|
||||
bucketName, objName, err)
|
||||
}
|
||||
|
||||
// Upload a part.
|
||||
partID := 1
|
||||
_, err = adminTestBed.objLayer.PutObjectPart(bucketName, objName, uploadID,
|
||||
partID, int64(len("hello")), bytes.NewReader([]byte("hello")), "", "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to upload part %d of %s/%s - %v", partID,
|
||||
bucketName, objName, err)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
object string
|
||||
dryrun string
|
||||
statusCode int
|
||||
}{
|
||||
// 1. Valid test case.
|
||||
{
|
||||
bucket: bucketName,
|
||||
object: objName,
|
||||
statusCode: http.StatusOK,
|
||||
},
|
||||
// 2. Invalid bucket name.
|
||||
{
|
||||
bucket: `invalid\\Bucket`,
|
||||
object: "myobject",
|
||||
statusCode: http.StatusBadRequest,
|
||||
},
|
||||
// 3. Bucket not found.
|
||||
{
|
||||
bucket: "bucketnotfound",
|
||||
object: "myobject",
|
||||
statusCode: http.StatusNotFound,
|
||||
},
|
||||
// 4. Invalid object name.
|
||||
{
|
||||
bucket: bucketName,
|
||||
object: `invalid\\Object`,
|
||||
statusCode: http.StatusBadRequest,
|
||||
},
|
||||
// 5. Object not found.
|
||||
{
|
||||
bucket: bucketName,
|
||||
object: "objectnotfound",
|
||||
statusCode: http.StatusNotFound,
|
||||
},
|
||||
// 6. Valid test case with dry-run.
|
||||
{
|
||||
bucket: bucketName,
|
||||
object: objName,
|
||||
dryrun: "yes",
|
||||
statusCode: http.StatusOK,
|
||||
},
|
||||
}
|
||||
for i, test := range testCases {
|
||||
// Prepare query params.
|
||||
queryVal := mkHealUploadQuery(test.bucket, test.object, uploadID, test.dryrun)
|
||||
req, err1 := buildAdminRequest(queryVal, "upload", http.MethodPost, 0, nil)
|
||||
if err1 != nil {
|
||||
t.Fatalf("Test %d - Failed to construct heal object request - %v", i+1, err1)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if test.statusCode != rec.Code {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
sample := testCases[0]
|
||||
// Modify authorization header after signing to test signature
|
||||
// mismatch handling.
|
||||
queryVal := mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
|
||||
req, err := buildAdminRequest(queryVal, "upload", "POST", 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct heal object request - %v", err)
|
||||
}
|
||||
|
||||
// Set x-amz-date to a date different than time of signing.
|
||||
req.Header.Set("x-amz-date", time.Time{}.Format(iso8601Format))
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusForbidden {
|
||||
t.Errorf("Expected %d but received %d", http.StatusBadRequest, rec.Code)
|
||||
}
|
||||
|
||||
// Set objectAPI to nil to test Server not initialized case.
|
||||
resetGlobalObjectAPI()
|
||||
queryVal = mkHealUploadQuery(sample.bucket, sample.object, uploadID, sample.dryrun)
|
||||
req, err = buildAdminRequest(queryVal, "upload", "POST", 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct heal object request - %v", err)
|
||||
}
|
||||
|
||||
rec = httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusServiceUnavailable {
|
||||
t.Errorf("Expected %d but received %d", http.StatusServiceUnavailable, rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
// TestHealFormatHandler - test for HealFormatHandler.
|
||||
@@ -1061,21 +1190,11 @@ func TestHealFormatHandler(t *testing.T) {
|
||||
// Prepare query params for heal-format mgmt REST API.
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("heal", "")
|
||||
req, err := newTestRequest("POST", "/?"+queryVal.Encode(), 0, nil)
|
||||
req, err := buildAdminRequest(queryVal, "format", "POST", 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct heal object request - %v", err)
|
||||
}
|
||||
|
||||
// Set x-minio-operation header to format.
|
||||
req.Header.Set(minioAdminOpHeader, "format")
|
||||
|
||||
// Sign the request using signature v4.
|
||||
cred := serverConfig.GetCredential()
|
||||
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to sign heal object request - %v", err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
@@ -1092,34 +1211,18 @@ func TestGetConfigHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// Prepare query params for get-config mgmt REST API.
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("config", "")
|
||||
|
||||
req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
|
||||
req, err := buildAdminRequest(queryVal, "get", http.MethodGet, 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct get-config object request - %v", err)
|
||||
}
|
||||
|
||||
// Set x-minio-operation header to get.
|
||||
req.Header.Set(minioAdminOpHeader, "get")
|
||||
|
||||
// Sign the request using signature v4.
|
||||
cred := serverConfig.GetCredential()
|
||||
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to sign heal object request - %v", err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
@@ -1137,14 +1240,8 @@ func TestSetConfigHandler(t *testing.T) {
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
eps, err := parseStorageEndpoints([]string{"http://127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse storage end point - %v", err)
|
||||
}
|
||||
|
||||
// Set globalMinioAddr to be able to distinguish local endpoints from remote.
|
||||
globalMinioAddr = eps[0].Host
|
||||
initGlobalAdminPeers(eps)
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// SetConfigHandler restarts minio setup - need to start a
|
||||
// signal receiver to receive on globalServiceSignalCh.
|
||||
@@ -1154,21 +1251,12 @@ func TestSetConfigHandler(t *testing.T) {
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("config", "")
|
||||
|
||||
req, err := newTestRequest("PUT", "/?"+queryVal.Encode(), int64(len(configJSON)), bytes.NewReader(configJSON))
|
||||
req, err := buildAdminRequest(queryVal, "set", http.MethodPut, int64(len(configJSON)),
|
||||
bytes.NewReader(configJSON))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct get-config object request - %v", err)
|
||||
}
|
||||
|
||||
// Set x-minio-operation header to set.
|
||||
req.Header.Set(minioAdminOpHeader, "set")
|
||||
|
||||
// Sign the request using signature v4.
|
||||
cred := serverConfig.GetCredential()
|
||||
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to sign heal object request - %v", err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
@@ -1186,6 +1274,58 @@ func TestSetConfigHandler(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminServerInfo(t *testing.T) {
|
||||
adminTestBed, err := prepareAdminXLTestBed()
|
||||
if err != nil {
|
||||
t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
|
||||
}
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
// Initialize admin peers to make admin RPC calls.
|
||||
globalMinioAddr = "127.0.0.1:9000"
|
||||
initGlobalAdminPeers(mustGetNewEndpointList("http://127.0.0.1:9000/d1"))
|
||||
|
||||
// Prepare query params for set-config mgmt REST API.
|
||||
queryVal := url.Values{}
|
||||
queryVal.Set("info", "")
|
||||
|
||||
req, err := buildAdminRequest(queryVal, "", http.MethodGet, 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct get-config object request - %v", err)
|
||||
}
|
||||
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
if rec.Code != http.StatusOK {
|
||||
t.Errorf("Expected to succeed but failed with %d", rec.Code)
|
||||
}
|
||||
|
||||
results := []ServerInfo{}
|
||||
err = json.NewDecoder(rec.Body).Decode(&results)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to decode set config result json %v", err)
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
t.Error("Expected at least one server info result")
|
||||
}
|
||||
|
||||
for _, serverInfo := range results {
|
||||
if len(serverInfo.Addr) == 0 {
|
||||
t.Error("Expected server address to be non empty")
|
||||
}
|
||||
if serverInfo.Error != "" {
|
||||
t.Errorf("Unexpected error = %v\n", serverInfo.Error)
|
||||
}
|
||||
if serverInfo.Data.StorageInfo.Free == 0 {
|
||||
t.Error("Expected StorageInfo.Free to be non empty")
|
||||
}
|
||||
if serverInfo.Data.Properties.Region != globalMinioDefaultRegion {
|
||||
t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, serverInfo.Data.Properties.Region)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestToAdminAPIErr - test for toAdminAPIErr helper function.
|
||||
func TestToAdminAPIErr(t *testing.T) {
|
||||
testCases := []struct {
|
||||
@@ -1219,6 +1359,11 @@ func TestToAdminAPIErr(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestWriteSetConfigResponse(t *testing.T) {
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
testCases := []struct {
|
||||
status bool
|
||||
errs []error
|
||||
@@ -1250,7 +1395,7 @@ func TestWriteSetConfigResponse(t *testing.T) {
|
||||
},
|
||||
}
|
||||
|
||||
testURL, err := url.Parse("dummy.com")
|
||||
testURL, err := url.Parse("http://dummy.com")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to parse a place-holder url")
|
||||
}
|
||||
@@ -1288,9 +1433,9 @@ func TestWriteSetConfigResponse(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// mkUploadsHealQuery - helper function to construct query values for
|
||||
// mkListUploadsHealQuery - helper function to construct query values for
|
||||
// listUploadsHeal.
|
||||
func mkUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
|
||||
func mkListUploadsHealQuery(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploadsStr string) url.Values {
|
||||
|
||||
queryVal := make(url.Values)
|
||||
queryVal.Set("heal", "")
|
||||
@@ -1310,7 +1455,7 @@ func TestListHealUploadsHandler(t *testing.T) {
|
||||
}
|
||||
defer adminTestBed.TearDown()
|
||||
|
||||
err = adminTestBed.objLayer.MakeBucket("mybucket")
|
||||
err = adminTestBed.objLayer.MakeBucketWithLocation("mybucket", "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to make bucket - %v", err)
|
||||
}
|
||||
@@ -1319,12 +1464,13 @@ func TestListHealUploadsHandler(t *testing.T) {
|
||||
defer adminTestBed.objLayer.DeleteBucket("mybucket")
|
||||
|
||||
testCases := []struct {
|
||||
bucket string
|
||||
prefix string
|
||||
keyMarker string
|
||||
delimiter string
|
||||
maxKeys string
|
||||
statusCode int
|
||||
bucket string
|
||||
prefix string
|
||||
keyMarker string
|
||||
delimiter string
|
||||
maxKeys string
|
||||
statusCode int
|
||||
expectedResp ListMultipartUploadsResponse
|
||||
}{
|
||||
// 1. Valid params.
|
||||
{
|
||||
@@ -1334,6 +1480,14 @@ func TestListHealUploadsHandler(t *testing.T) {
|
||||
delimiter: "/",
|
||||
maxKeys: "10",
|
||||
statusCode: http.StatusOK,
|
||||
expectedResp: ListMultipartUploadsResponse{
|
||||
XMLName: xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
|
||||
Bucket: "mybucket",
|
||||
KeyMarker: "prefix11",
|
||||
Delimiter: "/",
|
||||
Prefix: "prefix",
|
||||
MaxUploads: 10,
|
||||
},
|
||||
},
|
||||
// 2. Valid params with empty prefix.
|
||||
{
|
||||
@@ -1343,6 +1497,14 @@ func TestListHealUploadsHandler(t *testing.T) {
|
||||
delimiter: "/",
|
||||
maxKeys: "10",
|
||||
statusCode: http.StatusOK,
|
||||
expectedResp: ListMultipartUploadsResponse{
|
||||
XMLName: xml.Name{Space: "http://s3.amazonaws.com/doc/2006-03-01/", Local: "ListMultipartUploadsResult"},
|
||||
Bucket: "mybucket",
|
||||
KeyMarker: "",
|
||||
Delimiter: "/",
|
||||
Prefix: "",
|
||||
MaxUploads: 10,
|
||||
},
|
||||
},
|
||||
// 3. Invalid params with invalid bucket.
|
||||
{
|
||||
@@ -1401,23 +1563,64 @@ func TestListHealUploadsHandler(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
queryVal := mkUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
|
||||
queryVal := mkListUploadsHealQuery(test.bucket, test.prefix, test.keyMarker, "", test.delimiter, test.maxKeys)
|
||||
|
||||
req, err := newTestRequest("GET", "/?"+queryVal.Encode(), 0, nil)
|
||||
req, err := buildAdminRequest(queryVal, "list-uploads", http.MethodGet, 0, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d - Failed to construct list uploads needing heal request - %v", i+1, err)
|
||||
}
|
||||
req.Header.Set(minioAdminOpHeader, "list-uploads")
|
||||
|
||||
cred := serverConfig.GetCredential()
|
||||
err = signRequestV4(req, cred.AccessKey, cred.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d - Failed to sign list uploads needing heal request - %v", i+1, err)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
adminTestBed.mux.ServeHTTP(rec, req)
|
||||
|
||||
if test.statusCode != rec.Code {
|
||||
t.Errorf("Test %d - Expected HTTP status code %d but received %d", i+1, test.statusCode, rec.Code)
|
||||
}
|
||||
|
||||
// Compare result with the expected one only when we receive 200 OK
|
||||
if rec.Code == http.StatusOK {
|
||||
resp := rec.Result()
|
||||
xmlBytes, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Errorf("Test %d: Failed to read response %v", i+1, err)
|
||||
}
|
||||
|
||||
var actualResult ListMultipartUploadsResponse
|
||||
err = xml.Unmarshal(xmlBytes, &actualResult)
|
||||
if err != nil {
|
||||
t.Errorf("Test %d: Failed to unmarshal xml %v", i+1, err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(test.expectedResp, actualResult) {
|
||||
t.Fatalf("Test %d: Unexpected response `%+v`, expected: `%+v`", i+1, test.expectedResp, actualResult)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// Test for newHealResult helper function.
|
||||
func TestNewHealResult(t *testing.T) {
|
||||
testCases := []struct {
|
||||
healedDisks int
|
||||
offlineDisks int
|
||||
state healState
|
||||
}{
|
||||
// 1. No disks healed, no disks offline.
|
||||
{0, 0, healNone},
|
||||
// 2. No disks healed, non-zero disks offline.
|
||||
{0, 1, healNone},
|
||||
// 3. Non-zero disks healed, no disks offline.
|
||||
{1, 0, healOK},
|
||||
// 4. Non-zero disks healed, non-zero disks offline.
|
||||
{1, 1, healPartial},
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
actual := newHealResult(test.healedDisks, test.offlineDisks)
|
||||
if actual.State != test.state {
|
||||
t.Errorf("Test %d: Expected %v but received %v", i+1,
|
||||
test.state, actual.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -64,6 +64,8 @@ func registerAdminRouter(mux *router.Router) {
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
|
||||
// Heal Format.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
|
||||
// Heal Uploads.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "upload").HandlerFunc(adminAPI.HealUploadHandler)
|
||||
|
||||
/// Config operations
|
||||
|
||||
|
||||
+67
-51
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2014-2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2014, 2015, 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -20,7 +20,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -28,6 +28,8 @@ import (
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -35,7 +37,7 @@ const (
|
||||
serviceRestartRPC = "Admin.Restart"
|
||||
listLocksRPC = "Admin.ListLocks"
|
||||
reInitDisksRPC = "Admin.ReInitDisks"
|
||||
uptimeRPC = "Admin.Uptime"
|
||||
serverInfoDataRPC = "Admin.ServerInfoData"
|
||||
getConfigRPC = "Admin.GetConfig"
|
||||
writeTmpConfigRPC = "Admin.WriteTmpConfig"
|
||||
commitConfigRPC = "Admin.CommitConfig"
|
||||
@@ -57,7 +59,7 @@ type adminCmdRunner interface {
|
||||
Restart() error
|
||||
ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
|
||||
ReInitDisks() error
|
||||
Uptime() (time.Duration, error)
|
||||
ServerInfoData() (ServerInfoData, error)
|
||||
GetConfig() ([]byte, error)
|
||||
WriteTmpConfig(tmpFileName string, configBytes []byte) error
|
||||
CommitConfig(tmpFileName string) error
|
||||
@@ -110,26 +112,48 @@ func (rc remoteAdminClient) ReInitDisks() error {
|
||||
return rc.Call(reInitDisksRPC, &args, &reply)
|
||||
}
|
||||
|
||||
// Uptime - Returns the uptime of this server. Timestamp is taken
|
||||
// after object layer is initialized.
|
||||
func (lc localAdminClient) Uptime() (time.Duration, error) {
|
||||
// ServerInfoData - Returns the server info of this server.
|
||||
func (lc localAdminClient) ServerInfoData() (ServerInfoData, error) {
|
||||
if globalBootTime.IsZero() {
|
||||
return time.Duration(0), errServerNotInitialized
|
||||
return ServerInfoData{}, errServerNotInitialized
|
||||
}
|
||||
|
||||
return time.Now().UTC().Sub(globalBootTime), nil
|
||||
// Build storage info
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer == nil {
|
||||
return ServerInfoData{}, errServerNotInitialized
|
||||
}
|
||||
storage := objLayer.StorageInfo()
|
||||
|
||||
var arns []string
|
||||
for queueArn := range globalEventNotifier.GetAllExternalTargets() {
|
||||
arns = append(arns, queueArn)
|
||||
}
|
||||
|
||||
return ServerInfoData{
|
||||
StorageInfo: storage,
|
||||
ConnStats: globalConnStats.toServerConnStats(),
|
||||
HTTPStats: globalHTTPStats.toServerHTTPStats(),
|
||||
Properties: ServerProperties{
|
||||
Uptime: UTCNow().Sub(globalBootTime),
|
||||
Version: Version,
|
||||
CommitID: CommitID,
|
||||
SQSARN: arns,
|
||||
Region: serverConfig.GetRegion(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Uptime - returns the uptime of the server to which the RPC call is made.
|
||||
func (rc remoteAdminClient) Uptime() (time.Duration, error) {
|
||||
// ServerInfo - returns the server info of the server to which the RPC call is made.
|
||||
func (rc remoteAdminClient) ServerInfoData() (ServerInfoData, error) {
|
||||
args := AuthRPCArgs{}
|
||||
reply := UptimeReply{}
|
||||
err := rc.Call(uptimeRPC, &args, &reply)
|
||||
reply := ServerInfoDataReply{}
|
||||
err := rc.Call(serverInfoDataRPC, &args, &reply)
|
||||
if err != nil {
|
||||
return time.Duration(0), err
|
||||
return ServerInfoData{}, err
|
||||
}
|
||||
|
||||
return reply.Uptime, nil
|
||||
return reply.ServerInfoData, nil
|
||||
}
|
||||
|
||||
// GetConfig - returns config.json of the local server.
|
||||
@@ -211,52 +235,43 @@ type adminPeer struct {
|
||||
type adminPeers []adminPeer
|
||||
|
||||
// makeAdminPeers - helper function to construct a collection of adminPeer.
|
||||
func makeAdminPeers(eps []*url.URL) adminPeers {
|
||||
var servicePeers []adminPeer
|
||||
|
||||
// map to store peers that are already added to ret
|
||||
seenAddr := make(map[string]bool)
|
||||
|
||||
// add local (self) as peer in the array
|
||||
servicePeers = append(servicePeers, adminPeer{
|
||||
globalMinioAddr,
|
||||
func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
thisPeer := globalMinioAddr
|
||||
if globalMinioHost == "" {
|
||||
thisPeer = net.JoinHostPort("localhost", globalMinioPort)
|
||||
}
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
thisPeer,
|
||||
localAdminClient{},
|
||||
})
|
||||
seenAddr[globalMinioAddr] = true
|
||||
|
||||
serverCred := serverConfig.GetCredential()
|
||||
// iterate over endpoints to find new remote peers and add
|
||||
// them to ret.
|
||||
for _, ep := range eps {
|
||||
if ep.Host == "" {
|
||||
hostSet := set.CreateStringSet(globalMinioAddr)
|
||||
cred := serverConfig.GetCredential()
|
||||
serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
|
||||
for _, host := range GetRemotePeers(endpoints) {
|
||||
if hostSet.Contains(host) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if the remote host has been added already
|
||||
if !seenAddr[ep.Host] {
|
||||
cfg := authConfig{
|
||||
accessKey: serverCred.AccessKey,
|
||||
secretKey: serverCred.SecretKey,
|
||||
serverAddr: ep.Host,
|
||||
hostSet.Add(host)
|
||||
adminPeerList = append(adminPeerList, adminPeer{
|
||||
addr: host,
|
||||
cmdRunner: &remoteAdminClient{newAuthRPCClient(authConfig{
|
||||
accessKey: cred.AccessKey,
|
||||
secretKey: cred.SecretKey,
|
||||
serverAddr: host,
|
||||
serviceEndpoint: serviceEndpoint,
|
||||
secureConn: globalIsSSL,
|
||||
serviceEndpoint: path.Join(minioReservedBucketPath, adminPath),
|
||||
serviceName: "Admin",
|
||||
}
|
||||
|
||||
servicePeers = append(servicePeers, adminPeer{
|
||||
addr: ep.Host,
|
||||
cmdRunner: &remoteAdminClient{newAuthRPCClient(cfg)},
|
||||
})
|
||||
seenAddr[ep.Host] = true
|
||||
}
|
||||
})},
|
||||
})
|
||||
}
|
||||
|
||||
return servicePeers
|
||||
return adminPeerList
|
||||
}
|
||||
|
||||
// Initialize global adminPeer collection.
|
||||
func initGlobalAdminPeers(eps []*url.URL) {
|
||||
globalAdminPeers = makeAdminPeers(eps)
|
||||
func initGlobalAdminPeers(endpoints EndpointList) {
|
||||
globalAdminPeers = makeAdminPeers(endpoints)
|
||||
}
|
||||
|
||||
// invokeServiceCmd - Invoke Restart command.
|
||||
@@ -380,7 +395,7 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
|
||||
// the setup is the uptime of the single minio server
|
||||
// instance.
|
||||
if !globalIsDistXL {
|
||||
return time.Now().UTC().Sub(globalBootTime), nil
|
||||
return UTCNow().Sub(globalBootTime), nil
|
||||
}
|
||||
|
||||
uptimes := make(uptimeSlice, len(peers))
|
||||
@@ -391,7 +406,8 @@ func getPeerUptimes(peers adminPeers) (time.Duration, error) {
|
||||
wg.Add(1)
|
||||
go func(idx int, peer adminPeer) {
|
||||
defer wg.Done()
|
||||
uptimes[idx].uptime, uptimes[idx].err = peer.cmdRunner.Uptime()
|
||||
serverInfoData, rpcErr := peer.cmdRunner.ServerInfoData()
|
||||
uptimes[idx].uptime, uptimes[idx].err = serverInfoData.Properties.Uptime, rpcErr
|
||||
}(i, peer)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
@@ -245,7 +245,7 @@ func TestGetValidServerConfig(t *testing.T) {
|
||||
|
||||
// Invalid config - no quorum.
|
||||
serverConfigs = []serverConfigV13{c1, c2, c2, c1}
|
||||
validConfig, err = getValidServerConfig(serverConfigs, noErrs)
|
||||
_, err = getValidServerConfig(serverConfigs, noErrs)
|
||||
if err != errXLWriteQuorum {
|
||||
t.Errorf("Expected to fail due to lack of quorum but received %v", err)
|
||||
}
|
||||
@@ -253,7 +253,7 @@ func TestGetValidServerConfig(t *testing.T) {
|
||||
// All errors
|
||||
allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
|
||||
serverConfigs = []serverConfigV13{{}, {}, {}, {}}
|
||||
validConfig, err = getValidServerConfig(serverConfigs, allErrs)
|
||||
_, err = getValidServerConfig(serverConfigs, allErrs)
|
||||
if err != errXLWriteQuorum {
|
||||
t.Errorf("Expected to fail due to lack of quorum but received %v", err)
|
||||
}
|
||||
|
||||
+29
-12
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -53,10 +53,10 @@ type ListLocksReply struct {
|
||||
volLocks []VolumeLockInfo
|
||||
}
|
||||
|
||||
// UptimeReply - wraps the uptime response over RPC.
|
||||
type UptimeReply struct {
|
||||
// ServerInfoDataReply - wraps the server info response over RPC.
|
||||
type ServerInfoDataReply struct {
|
||||
AuthRPCReply
|
||||
Uptime time.Duration
|
||||
ServerInfoData ServerInfoData
|
||||
}
|
||||
|
||||
// ConfigReply - wraps the server config response over RPC.
|
||||
@@ -122,8 +122,8 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Uptime - returns the time when object layer was initialized on this server.
|
||||
func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
|
||||
// ServerInfo - returns the server info when object layer was initialized on this server.
|
||||
func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply) error {
|
||||
if err := args.IsAuthenticated(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -132,12 +132,29 @@ func (s *adminCmd) Uptime(args *AuthRPCArgs, reply *UptimeReply) error {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
// N B The uptime is computed assuming that the system time is
|
||||
// monotonic. This is not the case in time pkg in Go, see
|
||||
// https://github.com/golang/go/issues/12914. This is expected
|
||||
// to be fixed by go1.9.
|
||||
*reply = UptimeReply{
|
||||
Uptime: time.Now().UTC().Sub(globalBootTime),
|
||||
// Build storage info
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
storageInfo := objLayer.StorageInfo()
|
||||
|
||||
var arns []string
|
||||
for queueArn := range globalEventNotifier.GetAllExternalTargets() {
|
||||
arns = append(arns, queueArn)
|
||||
}
|
||||
|
||||
reply.ServerInfoData = ServerInfoData{
|
||||
Properties: ServerProperties{
|
||||
Uptime: UTCNow().Sub(globalBootTime),
|
||||
Version: Version,
|
||||
CommitID: CommitID,
|
||||
Region: serverConfig.GetRegion(),
|
||||
SQSARN: arns,
|
||||
},
|
||||
StorageInfo: storageInfo,
|
||||
ConnStats: globalConnStats.toServerConnStats(),
|
||||
HTTPStats: globalHTTPStats.toServerHTTPStats(),
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
@@ -18,9 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/url"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func testAdminCmd(cmd cmdType, t *testing.T) {
|
||||
@@ -40,7 +38,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
reply := LoginRPCReply{}
|
||||
err = adminServer.Login(&args, &reply)
|
||||
@@ -87,9 +85,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
defer removeRoots(xlDirs)
|
||||
|
||||
// Set globalEndpoints for a single node XL setup.
|
||||
for _, xlDir := range xlDirs {
|
||||
globalEndpoints = append(globalEndpoints, &url.URL{Path: xlDir})
|
||||
}
|
||||
globalEndpoints = mustGetNewEndpointList(xlDirs...)
|
||||
|
||||
// Setup admin rpc server for an XL backend.
|
||||
globalIsXL = true
|
||||
@@ -99,7 +95,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
reply := LoginRPCReply{}
|
||||
err = adminServer.Login(&args, &reply)
|
||||
@@ -124,7 +120,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
fsReply := LoginRPCReply{}
|
||||
err = fsAdminServer.Login(&fsArgs, &fsReply)
|
||||
@@ -161,7 +157,7 @@ func TestGetConfig(t *testing.T) {
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
reply := LoginRPCReply{}
|
||||
err = adminServer.Login(&args, &reply)
|
||||
@@ -205,7 +201,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
reply := LoginRPCReply{}
|
||||
err = adminServer.Login(&args, &reply)
|
||||
|
||||
+14
-1
@@ -114,6 +114,7 @@ const (
|
||||
ErrInvalidQueryParams
|
||||
ErrBucketAlreadyOwnedByYou
|
||||
ErrInvalidDuration
|
||||
ErrNotSupported
|
||||
// Add new error codes here.
|
||||
|
||||
// Bucket notification related errors.
|
||||
@@ -148,6 +149,7 @@ const (
|
||||
ErrAdminInvalidAccessKey
|
||||
ErrAdminInvalidSecretKey
|
||||
ErrAdminConfigNoQuorum
|
||||
ErrInsecureClientRequest
|
||||
)
|
||||
|
||||
// error code to APIError structure, these fields carry respective
|
||||
@@ -229,7 +231,7 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
},
|
||||
ErrInvalidAccessKeyID: {
|
||||
Code: "InvalidAccessKeyID",
|
||||
Code: "InvalidAccessKeyId",
|
||||
Description: "The access key ID you provided does not exist in our records.",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
@@ -617,6 +619,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Configuration update failed because server quorum was not met",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrInsecureClientRequest: {
|
||||
Code: "XMinioInsecureClientRequest",
|
||||
Description: "Cannot respond to plain-text request from TLS-encrypted server",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
|
||||
// Add your error structure here.
|
||||
}
|
||||
@@ -666,6 +673,8 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrInvalidBucketName
|
||||
case BucketNotFound:
|
||||
apiErr = ErrNoSuchBucket
|
||||
case BucketAlreadyOwnedByYou:
|
||||
apiErr = ErrBucketAlreadyOwnedByYou
|
||||
case BucketNotEmpty:
|
||||
apiErr = ErrBucketNotEmpty
|
||||
case BucketExists:
|
||||
@@ -698,8 +707,12 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrEntityTooLarge
|
||||
case ObjectTooSmall:
|
||||
apiErr = ErrEntityTooSmall
|
||||
case NotSupported:
|
||||
apiErr = ErrNotSupported
|
||||
case NotImplemented:
|
||||
apiErr = ErrNotImplemented
|
||||
case PolicyNotFound:
|
||||
apiErr = ErrNoSuchBucketPolicy
|
||||
default:
|
||||
apiErr = ErrInternalError
|
||||
}
|
||||
|
||||
@@ -103,6 +103,10 @@ func TestAPIErrCode(t *testing.T) {
|
||||
StorageFull{},
|
||||
ErrStorageFull,
|
||||
},
|
||||
{
|
||||
NotSupported{},
|
||||
ErrNotSupported,
|
||||
},
|
||||
{
|
||||
NotImplemented{},
|
||||
ErrNotImplemented,
|
||||
|
||||
+5
-4
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015,2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -34,8 +34,9 @@ func mustGetRequestID(t time.Time) string {
|
||||
// Write http common headers
|
||||
func setCommonHeaders(w http.ResponseWriter) {
|
||||
// Set unique request ID for each reply.
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(time.Now().UTC()))
|
||||
w.Header().Set(responseRequestIDKey, mustGetRequestID(UTCNow()))
|
||||
w.Header().Set("Server", globalServerUserAgent)
|
||||
w.Header().Set("X-Amz-Bucket-Region", serverConfig.GetRegion())
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
}
|
||||
|
||||
@@ -61,8 +62,8 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *h
|
||||
w.Header().Set("Last-Modified", lastModified)
|
||||
|
||||
// Set Etag if available.
|
||||
if objInfo.MD5Sum != "" {
|
||||
w.Header().Set("ETag", "\""+objInfo.MD5Sum+"\"")
|
||||
if objInfo.ETag != "" {
|
||||
w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
|
||||
}
|
||||
|
||||
// Set all other user defined metadata.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -18,12 +18,11 @@ package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNewRequestID(t *testing.T) {
|
||||
// Ensure that it returns an alphanumeric result of length 16.
|
||||
var id = mustGetRequestID(time.Now().UTC())
|
||||
var id = mustGetRequestID(UTCNow())
|
||||
|
||||
if len(id) != 16 {
|
||||
t.Fail()
|
||||
|
||||
+4
-4
@@ -321,8 +321,8 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
|
||||
}
|
||||
content.Key = object.Name
|
||||
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
|
||||
if object.MD5Sum != "" {
|
||||
content.ETag = "\"" + object.MD5Sum + "\""
|
||||
if object.ETag != "" {
|
||||
content.ETag = "\"" + object.ETag + "\""
|
||||
}
|
||||
content.Size = object.Size
|
||||
content.StorageClass = globalMinioDefaultStorageClass
|
||||
@@ -370,8 +370,8 @@ func generateListObjectsV2Response(bucket, prefix, token, startAfter, delimiter
|
||||
}
|
||||
content.Key = object.Name
|
||||
content.LastModified = object.ModTime.UTC().Format(timeFormatAMZLong)
|
||||
if object.MD5Sum != "" {
|
||||
content.ETag = "\"" + object.MD5Sum + "\""
|
||||
if object.ETag != "" {
|
||||
content.ETag = "\"" + object.ETag + "\""
|
||||
}
|
||||
content.Size = object.Size
|
||||
content.StorageClass = globalMinioDefaultStorageClass
|
||||
|
||||
+30
-27
@@ -64,7 +64,6 @@ func isRequestPostPolicySignatureV4(r *http.Request) bool {
|
||||
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
|
||||
func isRequestSignStreamingV4(r *http.Request) bool {
|
||||
return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
|
||||
r.Header.Get("content-encoding") == streamingContentEncoding &&
|
||||
r.Method == httpPUT
|
||||
}
|
||||
|
||||
@@ -114,13 +113,13 @@ func checkRequestAuthType(r *http.Request, bucket, policyAction, region string)
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
}
|
||||
return s3Error
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, region)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
}
|
||||
return s3Error
|
||||
}
|
||||
@@ -143,19 +142,16 @@ func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) {
|
||||
return doesPresignV2SignatureMatch(r)
|
||||
}
|
||||
|
||||
func reqSignatureV4Verify(r *http.Request) (s3Error APIErrorCode) {
|
||||
sha256sum := r.Header.Get("X-Amz-Content-Sha256")
|
||||
// Skips calculating sha256 on the payload on server,
|
||||
// if client requested for it.
|
||||
if skipContentSha256Cksum(r) {
|
||||
sha256sum = unsignedPayload
|
||||
func reqSignatureV4Verify(r *http.Request, region string) (s3Error APIErrorCode) {
|
||||
sha256sum := getContentSha256Cksum(r)
|
||||
switch {
|
||||
case isRequestSignatureV4(r):
|
||||
return doesSignatureMatch(sha256sum, r, region)
|
||||
case isRequestPresignedSignatureV4(r):
|
||||
return doesPresignedSignatureMatch(sha256sum, r, region)
|
||||
default:
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if isRequestSignatureV4(r) {
|
||||
return doesSignatureMatch(sha256sum, r, serverConfig.GetRegion())
|
||||
} else if isRequestPresignedSignatureV4(r) {
|
||||
return doesPresignedSignatureMatch(sha256sum, r, serverConfig.GetRegion())
|
||||
}
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
// Verify if request has valid AWS Signature Version '4'.
|
||||
@@ -163,32 +159,39 @@ func isReqAuthenticated(r *http.Request, region string) (s3Error APIErrorCode) {
|
||||
if r == nil {
|
||||
return ErrInternalError
|
||||
}
|
||||
if errCode := reqSignatureV4Verify(r, region); errCode != ErrNone {
|
||||
return errCode
|
||||
}
|
||||
payload, err := ioutil.ReadAll(r.Body)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read request body for signature verification")
|
||||
return ErrInternalError
|
||||
}
|
||||
|
||||
// Populate back the payload.
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
|
||||
|
||||
// Verify Content-Md5, if payload is set.
|
||||
if r.Header.Get("Content-Md5") != "" {
|
||||
if r.Header.Get("Content-Md5") != getMD5HashBase64(payload) {
|
||||
return ErrBadDigest
|
||||
}
|
||||
}
|
||||
// Populate back the payload.
|
||||
r.Body = ioutil.NopCloser(bytes.NewReader(payload))
|
||||
// Skips calculating sha256 on the payload on server, if client requested for it.
|
||||
var sha256sum string
|
||||
|
||||
if skipContentSha256Cksum(r) {
|
||||
sha256sum = unsignedPayload
|
||||
} else {
|
||||
sha256sum = getSHA256Hash(payload)
|
||||
return ErrNone
|
||||
}
|
||||
if isRequestSignatureV4(r) {
|
||||
return doesSignatureMatch(sha256sum, r, region)
|
||||
} else if isRequestPresignedSignatureV4(r) {
|
||||
return doesPresignedSignatureMatch(sha256sum, r, region)
|
||||
|
||||
// Verify that X-Amz-Content-Sha256 Header == sha256(payload)
|
||||
// If X-Amz-Content-Sha256 header is not sent then we don't calculate/verify sha256(payload)
|
||||
sum := r.Header.Get("X-Amz-Content-Sha256")
|
||||
if isRequestPresignedSignatureV4(r) {
|
||||
sum = r.URL.Query().Get("X-Amz-Content-Sha256")
|
||||
}
|
||||
return ErrAccessDenied
|
||||
if sum != "" && sum != getSHA256Hash(payload) {
|
||||
return ErrContentSHA256Mismatch
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// authHandler - handles all the incoming authorization headers and validates them if possible.
|
||||
|
||||
@@ -308,6 +308,16 @@ func mustNewSignedRequest(method string, urlStr string, contentLength int64, bod
|
||||
return req
|
||||
}
|
||||
|
||||
func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
|
||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||
req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
|
||||
cred := serverConfig.GetCredential()
|
||||
if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
|
||||
t.Fatalf("Unable to initialized new signed http request %s", err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// Tests is requested authenticated function, tests replies for s3 errors.
|
||||
func TestIsReqAuthenticated(t *testing.T) {
|
||||
path, err := newTestConfig(globalMinioDefaultRegion)
|
||||
@@ -333,16 +343,13 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
// When request is unsigned, access denied is returned.
|
||||
{mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied},
|
||||
// When request is properly signed, but has bad Content-MD5 header.
|
||||
{mustNewSignedRequest("PUT", "http://127.0.0.1:9000", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
|
||||
{mustNewSignedBadMD5Request("PUT", "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest},
|
||||
// When request is properly signed, error is none.
|
||||
{mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrNone},
|
||||
}
|
||||
|
||||
// Validates all testcases.
|
||||
for _, testCase := range testCases {
|
||||
if testCase.s3Error == ErrBadDigest {
|
||||
testCase.req.Header.Set("Content-Md5", "garbage")
|
||||
}
|
||||
if s3Error := isReqAuthenticated(testCase.req, serverConfig.GetRegion()); s3Error != testCase.s3Error {
|
||||
t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
|
||||
}
|
||||
|
||||
+47
-37
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -52,10 +52,10 @@ type authConfig struct {
|
||||
|
||||
// AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
|
||||
type AuthRPCClient struct {
|
||||
sync.Mutex // Mutex to lock this object.
|
||||
rpcClient *RPCClient // Reconnectable RPC client to make any RPC call.
|
||||
config authConfig // Authentication configuration information.
|
||||
authToken string // Authentication token.
|
||||
sync.RWMutex // Mutex to lock this object.
|
||||
rpcClient *RPCClient // Reconnectable RPC client to make any RPC call.
|
||||
config authConfig // Authentication configuration information.
|
||||
authToken string // Authentication token.
|
||||
}
|
||||
|
||||
// newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
|
||||
@@ -78,33 +78,43 @@ func newAuthRPCClient(config authConfig) *AuthRPCClient {
|
||||
}
|
||||
}
|
||||
|
||||
// Login - a jwt based authentication is performed with rpc server.
|
||||
// Login a JWT based authentication is performed with rpc server.
|
||||
func (authClient *AuthRPCClient) Login() (err error) {
|
||||
// Login should be attempted one at a time.
|
||||
//
|
||||
// The reason for large region lock here is
|
||||
// to avoid two simultaneous login attempts
|
||||
// racing over each other.
|
||||
//
|
||||
// #1 Login() gets the lock proceeds to login.
|
||||
// #2 Login() waits for the unlock to happen
|
||||
// after login in #1.
|
||||
// #1 Successfully completes login saves the
|
||||
// newly acquired token.
|
||||
// #2 Successfully gets the lock and proceeds,
|
||||
// but since we have acquired the token
|
||||
// already the call quickly returns.
|
||||
authClient.Lock()
|
||||
defer authClient.Unlock()
|
||||
|
||||
// Return if already logged in.
|
||||
if authClient.authToken != "" {
|
||||
return nil
|
||||
// Attempt to login if not logged in already.
|
||||
if authClient.authToken == "" {
|
||||
// Login to authenticate and acquire a new auth token.
|
||||
var (
|
||||
loginMethod = authClient.config.serviceName + loginMethodName
|
||||
loginArgs = LoginRPCArgs{
|
||||
Username: authClient.config.accessKey,
|
||||
Password: authClient.config.secretKey,
|
||||
Version: Version,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
loginReply = LoginRPCReply{}
|
||||
)
|
||||
if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &loginReply); err != nil {
|
||||
return err
|
||||
}
|
||||
authClient.authToken = loginReply.AuthToken
|
||||
}
|
||||
|
||||
// Call login.
|
||||
args := LoginRPCArgs{
|
||||
Username: authClient.config.accessKey,
|
||||
Password: authClient.config.secretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
}
|
||||
|
||||
reply := LoginRPCReply{}
|
||||
serviceMethod := authClient.config.serviceName + loginMethodName
|
||||
if err = authClient.rpcClient.Call(serviceMethod, &args, &reply); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Logged in successfully.
|
||||
authClient.authToken = reply.AuthToken
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -112,17 +122,17 @@ func (authClient *AuthRPCClient) Login() (err error) {
|
||||
func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
|
||||
SetAuthToken(authToken string)
|
||||
}, reply interface{}) (err error) {
|
||||
// On successful login, execute RPC call.
|
||||
if err = authClient.Login(); err == nil {
|
||||
authClient.Lock()
|
||||
// Set token and timestamp before the rpc call.
|
||||
args.SetAuthToken(authClient.authToken)
|
||||
authClient.Unlock()
|
||||
if err = authClient.Login(); err != nil {
|
||||
return err
|
||||
} // On successful login, execute RPC call.
|
||||
|
||||
// Do RPC call.
|
||||
err = authClient.rpcClient.Call(serviceMethod, args, reply)
|
||||
}
|
||||
return err
|
||||
authClient.RLock()
|
||||
// Set token before the rpc call.
|
||||
args.SetAuthToken(authClient.authToken)
|
||||
authClient.RUnlock()
|
||||
|
||||
// Do an RPC call.
|
||||
return authClient.rpcClient.Call(serviceMethod, args, reply)
|
||||
}
|
||||
|
||||
// Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.
|
||||
|
||||
+28
-29
@@ -23,7 +23,6 @@ import (
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
)
|
||||
@@ -40,7 +39,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(bucket)
|
||||
err = obj.MakeBucketWithLocation(bucket, "")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -50,7 +49,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = getMD5Hash(textData)
|
||||
metadata["etag"] = getMD5Hash(textData)
|
||||
sha256sum := ""
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
@@ -62,8 +61,8 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.MD5Sum != metadata["md5Sum"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
|
||||
}
|
||||
}
|
||||
// Benchmark ends here. Stop timer.
|
||||
@@ -79,22 +78,22 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
object := getRandomObjectName()
|
||||
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(bucket)
|
||||
err = obj.MakeBucketWithLocation(bucket, "")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
objSize := 128 * humanize.MiByte
|
||||
|
||||
// PutObjectPart returns md5Sum of the object inserted.
|
||||
// md5Sum variable is assigned with that value.
|
||||
var md5Sum, uploadID string
|
||||
// PutObjectPart returns etag of the object inserted.
|
||||
// etag variable is assigned with that value.
|
||||
var etag, uploadID string
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for NewMultipartUpload.
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = getMD5Hash(textData)
|
||||
metadata["etag"] = getMD5Hash(textData)
|
||||
sha256sum := ""
|
||||
uploadID, err = obj.NewMultipartUpload(bucket, object, metadata)
|
||||
if err != nil {
|
||||
@@ -116,14 +115,14 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
textPartData = textData[j*partSize:]
|
||||
}
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = getMD5Hash([]byte(textPartData))
|
||||
metadata["etag"] = getMD5Hash([]byte(textPartData))
|
||||
var partInfo PartInfo
|
||||
partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["md5Sum"], sha256sum)
|
||||
partInfo, err = obj.PutObjectPart(bucket, object, uploadID, j, int64(len(textPartData)), bytes.NewBuffer(textPartData), metadata["etag"], sha256sum)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if partInfo.ETag != metadata["md5Sum"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, md5Sum, metadata["md5Sum"])
|
||||
if partInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, metadata["etag"])
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -200,7 +199,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(bucket)
|
||||
err = obj.MakeBucketWithLocation(bucket, "")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -209,19 +208,19 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
for i := 0; i < 10; i++ {
|
||||
// get text data generated for number of bytes equal to object size.
|
||||
textData := generateBytesData(objSize)
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
// generate etag for the generated data.
|
||||
// etag of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = getMD5Hash(textData)
|
||||
metadata["etag"] = getMD5Hash(textData)
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(bucket, "object"+strconv.Itoa(i), int64(len(textData)), bytes.NewBuffer(textData), metadata, sha256sum)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.MD5Sum != metadata["md5Sum"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,7 +244,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
func getRandomByte() []byte {
|
||||
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
// seeding the random number generator.
|
||||
rand.Seed(time.Now().UTC().UnixNano())
|
||||
rand.Seed(UTCNow().UnixNano())
|
||||
var b byte
|
||||
// pick a character randomly.
|
||||
b = letterBytes[rand.Intn(len(letterBytes))]
|
||||
@@ -308,7 +307,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(bucket)
|
||||
err = obj.MakeBucketWithLocation(bucket, "")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -318,7 +317,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// generate md5sum for the generated data.
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = getMD5Hash([]byte(textData))
|
||||
metadata["etag"] = getMD5Hash([]byte(textData))
|
||||
sha256sum := ""
|
||||
// benchmark utility which helps obtain number of allocations and bytes allocated per ops.
|
||||
b.ReportAllocs()
|
||||
@@ -333,8 +332,8 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.MD5Sum != metadata["md5Sum"] {
|
||||
b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.MD5Sum, metadata["md5Sum"])
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, metadata["etag"])
|
||||
}
|
||||
i++
|
||||
}
|
||||
@@ -356,7 +355,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// obtains random bucket name.
|
||||
bucket := getRandomBucketName()
|
||||
// create bucket.
|
||||
err = obj.MakeBucket(bucket)
|
||||
err = obj.MakeBucketWithLocation(bucket, "")
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -368,7 +367,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// md5sum of the data to written is required as input for PutObject.
|
||||
// PutObject is the functions which writes the data onto the FS/XL backend.
|
||||
metadata := make(map[string]string)
|
||||
metadata["md5Sum"] = getMD5Hash([]byte(textData))
|
||||
metadata["etag"] = getMD5Hash([]byte(textData))
|
||||
sha256sum := ""
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
@@ -376,8 +375,8 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
if objInfo.MD5Sum != metadata["md5Sum"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.MD5Sum, metadata["md5Sum"])
|
||||
if objInfo.ETag != metadata["etag"] {
|
||||
b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, metadata["etag"])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// BrowserFlag - wrapper bool type.
|
||||
type BrowserFlag bool
|
||||
|
||||
// String - returns string of BrowserFlag.
|
||||
func (bf BrowserFlag) String() string {
|
||||
if bf {
|
||||
return "on"
|
||||
}
|
||||
|
||||
return "off"
|
||||
}
|
||||
|
||||
// MarshalJSON - converts BrowserFlag into JSON data.
|
||||
func (bf BrowserFlag) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(bf.String())
|
||||
}
|
||||
|
||||
// UnmarshalJSON - parses given data into BrowserFlag.
|
||||
func (bf *BrowserFlag) UnmarshalJSON(data []byte) (err error) {
|
||||
var s string
|
||||
if err = json.Unmarshal(data, &s); err == nil {
|
||||
b := BrowserFlag(true)
|
||||
if s == "" {
|
||||
// Empty string is treated as valid.
|
||||
*bf = b
|
||||
} else if b, err = ParseBrowserFlag(s); err == nil {
|
||||
*bf = b
|
||||
}
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// ParseBrowserFlag - parses string into BrowserFlag.
|
||||
func ParseBrowserFlag(s string) (bf BrowserFlag, err error) {
|
||||
if s == "on" {
|
||||
bf = true
|
||||
} else if s == "off" {
|
||||
bf = false
|
||||
} else {
|
||||
err = fmt.Errorf("invalid value ‘%s’ for BrowserFlag", s)
|
||||
}
|
||||
|
||||
return bf, err
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Test BrowserFlag.String()
|
||||
func TestBrowserFlagString(t *testing.T) {
|
||||
var bf BrowserFlag
|
||||
|
||||
testCases := []struct {
|
||||
flag BrowserFlag
|
||||
expectedResult string
|
||||
}{
|
||||
{bf, "off"},
|
||||
{BrowserFlag(true), "on"},
|
||||
{BrowserFlag(false), "off"},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
str := testCase.flag.String()
|
||||
if testCase.expectedResult != str {
|
||||
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, str)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test BrowserFlag.MarshalJSON()
|
||||
func TestBrowserFlagMarshalJSON(t *testing.T) {
|
||||
var bf BrowserFlag
|
||||
|
||||
testCases := []struct {
|
||||
flag BrowserFlag
|
||||
expectedResult string
|
||||
}{
|
||||
{bf, `"off"`},
|
||||
{BrowserFlag(true), `"on"`},
|
||||
{BrowserFlag(false), `"off"`},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
data, _ := testCase.flag.MarshalJSON()
|
||||
if testCase.expectedResult != string(data) {
|
||||
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, string(data))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test BrowserFlag.UnmarshalJSON()
|
||||
func TestBrowserFlagUnmarshalJSON(t *testing.T) {
|
||||
testCases := []struct {
|
||||
data []byte
|
||||
expectedResult BrowserFlag
|
||||
expectedErr error
|
||||
}{
|
||||
{[]byte(`{}`), BrowserFlag(false), errors.New("json: cannot unmarshal object into Go value of type string")},
|
||||
{[]byte(`["on"]`), BrowserFlag(false), errors.New("json: cannot unmarshal array into Go value of type string")},
|
||||
{[]byte(`"junk"`), BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
|
||||
{[]byte(`"true"`), BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
|
||||
{[]byte(`"false"`), BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
|
||||
{[]byte(`"ON"`), BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
|
||||
{[]byte(`"OFF"`), BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
|
||||
{[]byte(`""`), BrowserFlag(true), nil},
|
||||
{[]byte(`"on"`), BrowserFlag(true), nil},
|
||||
{[]byte(`"off"`), BrowserFlag(false), nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
var flag BrowserFlag
|
||||
err := (&flag).UnmarshalJSON(testCase.data)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
|
||||
if err == nil && testCase.expectedResult != flag {
|
||||
t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, flag)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test ParseBrowserFlag()
|
||||
func TestParseBrowserFlag(t *testing.T) {
|
||||
testCases := []struct {
|
||||
flagStr string
|
||||
expectedResult BrowserFlag
|
||||
expectedErr error
|
||||
}{
|
||||
{"", BrowserFlag(false), errors.New("invalid value ‘’ for BrowserFlag")},
|
||||
{"junk", BrowserFlag(false), errors.New("invalid value ‘junk’ for BrowserFlag")},
|
||||
{"true", BrowserFlag(false), errors.New("invalid value ‘true’ for BrowserFlag")},
|
||||
{"false", BrowserFlag(false), errors.New("invalid value ‘false’ for BrowserFlag")},
|
||||
{"ON", BrowserFlag(false), errors.New("invalid value ‘ON’ for BrowserFlag")},
|
||||
{"OFF", BrowserFlag(false), errors.New("invalid value ‘OFF’ for BrowserFlag")},
|
||||
{"on", BrowserFlag(true), nil},
|
||||
{"off", BrowserFlag(false), nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
bf, err := ParseBrowserFlag(testCase.flagStr)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
|
||||
if err == nil && testCase.expectedResult != bf {
|
||||
t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, bf)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"path"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// API suite container common to both FS and XL.
|
||||
@@ -96,7 +95,7 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
rreply := &LoginRPCReply{}
|
||||
err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
|
||||
@@ -111,7 +110,7 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
|
||||
Username: creds.AccessKey,
|
||||
Password: creds.SecretKey,
|
||||
Version: Version,
|
||||
RequestTime: time.Now().UTC(),
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
rreply = &LoginRPCReply{}
|
||||
err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
|
||||
|
||||
+37
-6
@@ -20,6 +20,7 @@ import (
|
||||
"encoding/base64"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
@@ -82,6 +83,10 @@ func enforceBucketPolicy(bucket, action, resource, referer string, queryParams u
|
||||
|
||||
// Check if the action is allowed on the bucket/prefix.
|
||||
func isBucketActionAllowed(action, bucket, prefix string) bool {
|
||||
if globalBucketPolicies == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
policy := globalBucketPolicies.GetBucketPolicy(bucket)
|
||||
if policy == nil {
|
||||
return false
|
||||
@@ -322,6 +327,13 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
// Write success response.
|
||||
writeSuccessResponseXML(w, encodedSuccessResponse)
|
||||
|
||||
// Get host and port from Request.RemoteAddr failing which
|
||||
// fill them with empty strings.
|
||||
host, port, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
host, port = "", ""
|
||||
}
|
||||
|
||||
// Notify deleted event for objects.
|
||||
for _, dobj := range deletedObjects {
|
||||
eventNotify(eventData{
|
||||
@@ -331,6 +343,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
Name: dobj.ObjectName,
|
||||
},
|
||||
ReqParams: extractReqParams(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: host,
|
||||
Port: port,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -359,19 +374,26 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// Validate if incoming location constraint is valid, reject
|
||||
// requests which do not follow valid region requirements.
|
||||
if s3Error := isValidLocationConstraint(r); s3Error != ErrNone {
|
||||
// Parse incoming location constraint.
|
||||
location, s3Error := parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Validate if location sent by the client is valid, reject
|
||||
// requests which do not follow valid region requirements.
|
||||
if !isValidLocation(location) {
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
bucketLock := globalNSMutex.NewNSLock(bucket, "")
|
||||
bucketLock.Lock()
|
||||
defer bucketLock.Unlock()
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucket(bucket)
|
||||
err := objectAPI.MakeBucketWithLocation(bucket, "")
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to create a bucket.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
@@ -517,15 +539,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("ETag", `"`+objInfo.MD5Sum+`"`)
|
||||
w.Header().Set("ETag", `"`+objInfo.ETag+`"`)
|
||||
w.Header().Set("Location", getObjectLocation(bucket, object))
|
||||
|
||||
// Get host and port from Request.RemoteAddr.
|
||||
host, port, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
host, port = "", ""
|
||||
}
|
||||
|
||||
// Notify object created event.
|
||||
defer eventNotify(eventData{
|
||||
Type: ObjectCreatedPost,
|
||||
Bucket: objInfo.Bucket,
|
||||
ObjInfo: objInfo,
|
||||
ReqParams: extractReqParams(r),
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: host,
|
||||
Port: port,
|
||||
})
|
||||
|
||||
if successRedirect != "" {
|
||||
@@ -541,7 +572,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
resp := encodeResponse(PostResponse{
|
||||
Bucket: objInfo.Bucket,
|
||||
Key: objInfo.Name,
|
||||
ETag: `"` + objInfo.MD5Sum + `"`,
|
||||
ETag: `"` + objInfo.ETag + `"`,
|
||||
Location: getObjectLocation(objInfo.Bucket, objInfo.Name),
|
||||
})
|
||||
writeResponse(w, http.StatusCreated, resp, "application/xml")
|
||||
|
||||
@@ -68,7 +68,7 @@ func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName stri
|
||||
locationResponse: []byte(""),
|
||||
errorResponse: APIErrorResponse{
|
||||
Resource: "/" + bucketName + "/",
|
||||
Code: "InvalidAccessKeyID",
|
||||
Code: "InvalidAccessKeyId",
|
||||
Message: "The access key ID you provided does not exist in our records.",
|
||||
},
|
||||
shouldPass: false,
|
||||
@@ -771,3 +771,36 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
// `ExecObjectLayerAPINilTest` manages the operation.
|
||||
ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
|
||||
}
|
||||
|
||||
func TestIsBucketActionAllowed(t *testing.T) {
|
||||
ExecObjectLayerAPITest(t, testIsBucketActionAllowedHandler, []string{"BucketLocation"})
|
||||
}
|
||||
|
||||
func testIsBucketActionAllowedHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials credential, t *testing.T) {
|
||||
|
||||
testCases := []struct {
|
||||
// input.
|
||||
action string
|
||||
bucket string
|
||||
prefix string
|
||||
isGlobalPoliciesNil bool
|
||||
// flag indicating whether the test should pass.
|
||||
shouldPass bool
|
||||
}{
|
||||
{"s3:GetBucketLocation", "mybucket", "abc", true, false},
|
||||
{"s3:ListObject", "mybucket", "abc", false, false},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
if testCase.isGlobalPoliciesNil {
|
||||
globalBucketPolicies = nil
|
||||
} else {
|
||||
initBucketPolicies(obj)
|
||||
}
|
||||
isAllowed := isBucketActionAllowed(testCase.action, testCase.bucket, testCase.prefix)
|
||||
if isAllowed != testCase.shouldPass {
|
||||
t.Errorf("Case %d: Expected the response status to be `%t`, but instead found `%t`", i+1, testCase.shouldPass, isAllowed)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -94,6 +94,10 @@ const (
|
||||
ObjectCreatedCompleteMultipartUpload
|
||||
// ObjectRemovedDelete is s3:ObjectRemoved:Delete
|
||||
ObjectRemovedDelete
|
||||
// ObjectAccessedGet is s3:ObjectAccessed:Get
|
||||
ObjectAccessedGet
|
||||
// ObjectAccessedHead is s3:ObjectAccessed:Head
|
||||
ObjectAccessedHead
|
||||
)
|
||||
|
||||
// Stringer interface for event name.
|
||||
@@ -109,6 +113,10 @@ func (eventName EventName) String() string {
|
||||
return "s3:ObjectCreated:CompleteMultipartUpload"
|
||||
case ObjectRemovedDelete:
|
||||
return "s3:ObjectRemoved:Delete"
|
||||
case ObjectAccessedGet:
|
||||
return "s3:ObjectAccessed:Get"
|
||||
case ObjectAccessedHead:
|
||||
return "s3:ObjectAccessed:Head"
|
||||
default:
|
||||
return "s3:Unknown"
|
||||
}
|
||||
@@ -128,11 +136,13 @@ type bucketMeta struct {
|
||||
|
||||
// Notification event object metadata.
|
||||
type objectMeta struct {
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
ETag string `json:"eTag,omitempty"`
|
||||
VersionID string `json:"versionId,omitempty"`
|
||||
Sequencer string `json:"sequencer"`
|
||||
Key string `json:"key"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
ETag string `json:"eTag,omitempty"`
|
||||
ContentType string `json:"contentType,omitempty"`
|
||||
UserDefined map[string]string `json:"userDefined,omitempty"`
|
||||
VersionID string `json:"versionId,omitempty"`
|
||||
Sequencer string `json:"sequencer"`
|
||||
}
|
||||
|
||||
const (
|
||||
@@ -168,6 +178,14 @@ const (
|
||||
eventVersion = "2.0"
|
||||
)
|
||||
|
||||
// sourceInfo represents information on the client that triggered the
|
||||
// event notification.
|
||||
type sourceInfo struct {
|
||||
Host string `json:"host"`
|
||||
Port string `json:"port"`
|
||||
UserAgent string `json:"userAgent"`
|
||||
}
|
||||
|
||||
// NotificationEvent represents an Amazon an S3 bucket notification event.
|
||||
type NotificationEvent struct {
|
||||
EventVersion string `json:"eventVersion"`
|
||||
@@ -179,6 +197,7 @@ type NotificationEvent struct {
|
||||
RequestParameters map[string]string `json:"requestParameters"`
|
||||
ResponseElements map[string]string `json:"responseElements"`
|
||||
S3 eventMeta `json:"s3"`
|
||||
Source sourceInfo `json:"source"`
|
||||
}
|
||||
|
||||
// Represents the minio sqs type and account id's.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -285,7 +285,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
return
|
||||
}
|
||||
|
||||
accountID := fmt.Sprintf("%d", time.Now().UTC().UnixNano())
|
||||
accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
|
||||
accountARN := fmt.Sprintf(
|
||||
"%s:%s:%s:%s-%s",
|
||||
minioTopic,
|
||||
|
||||
@@ -268,6 +268,7 @@ func testListenBucketNotificationNilHandler(obj ObjectLayer, instanceType, bucke
|
||||
[]string{"*.jpg"}, []string{
|
||||
"s3:ObjectCreated:*",
|
||||
"s3:ObjectRemoved:*",
|
||||
"s3:ObjectAccessed:*",
|
||||
}), 0, nil, credentials.AccessKey, credentials.SecretKey)
|
||||
if tErr != nil {
|
||||
t.Fatalf("%s: Failed to create HTTP testRequest for ListenBucketNotification: <ERROR> %v", instanceType, tErr)
|
||||
|
||||
@@ -16,7 +16,12 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "strings"
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
// List of valid event types.
|
||||
var suppportedEventTypes = map[string]struct{}{
|
||||
@@ -29,6 +34,9 @@ var suppportedEventTypes = map[string]struct{}{
|
||||
// Object removed event types.
|
||||
"s3:ObjectRemoved:*": {},
|
||||
"s3:ObjectRemoved:Delete": {},
|
||||
"s3:ObjectAccessed:Get": {},
|
||||
"s3:ObjectAccessed:Head": {},
|
||||
"s3:ObjectAccessed:*": {},
|
||||
}
|
||||
|
||||
// checkEvent - checks if an event is supported.
|
||||
@@ -104,20 +112,21 @@ func checkARN(arn, arnType string) APIErrorCode {
|
||||
if !strings.HasPrefix(arn, arnType) {
|
||||
return ErrARNNotification
|
||||
}
|
||||
if !strings.HasPrefix(arn, arnType+serverConfig.GetRegion()+":") {
|
||||
return ErrRegionNotification
|
||||
}
|
||||
account := strings.SplitN(strings.TrimPrefix(arn, arnType+serverConfig.GetRegion()+":"), ":", 2)
|
||||
switch len(account) {
|
||||
case 1:
|
||||
// This means ARN is malformed, account should have min of 2elements.
|
||||
strs := strings.SplitN(arn, ":", -1)
|
||||
if len(strs) != 6 {
|
||||
return ErrARNNotification
|
||||
case 2:
|
||||
// Account topic id or topic name cannot be empty.
|
||||
if account[0] == "" || account[1] == "" {
|
||||
return ErrARNNotification
|
||||
}
|
||||
if serverConfig.GetRegion() != "" {
|
||||
region := strs[3]
|
||||
if region != serverConfig.GetRegion() {
|
||||
return ErrRegionNotification
|
||||
}
|
||||
}
|
||||
accountID := strs[4]
|
||||
resource := strs[5]
|
||||
if accountID == "" || resource == "" {
|
||||
return ErrARNNotification
|
||||
}
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
@@ -148,6 +157,10 @@ func isValidQueueID(queueARN string) bool {
|
||||
pgN := serverConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
|
||||
// Postgres can work with only default conn. info.
|
||||
return pgN.Enable
|
||||
} else if isMySQLQueue(sqsARN) {
|
||||
msqlN := serverConfig.Notify.GetMySQLByID(sqsARN.AccountID)
|
||||
// Mysql can work with only default conn. info.
|
||||
return msqlN.Enable
|
||||
} else if isKafkaQueue(sqsARN) {
|
||||
kafkaN := serverConfig.Notify.GetKafkaByID(sqsARN.AccountID)
|
||||
return (kafkaN.Enable && len(kafkaN.Brokers) > 0 &&
|
||||
@@ -200,16 +213,14 @@ func validateQueueConfigs(queueConfigs []queueConfig) APIErrorCode {
|
||||
|
||||
// Check all the queue configs for any duplicates.
|
||||
func checkDuplicateQueueConfigs(configs []queueConfig) APIErrorCode {
|
||||
var queueConfigARNS []string
|
||||
queueConfigARNS := set.NewStringSet()
|
||||
|
||||
// Navigate through each configs and count the entries.
|
||||
for _, config := range configs {
|
||||
queueConfigARNS = append(queueConfigARNS, config.QueueARN)
|
||||
queueConfigARNS.Add(config.QueueARN)
|
||||
}
|
||||
|
||||
// Check if there are any duplicate counts.
|
||||
if err := checkDuplicateStrings(queueConfigARNS); err != nil {
|
||||
errorIf(err, "Invalid queue configs found.")
|
||||
if len(queueConfigARNS) != len(configs) {
|
||||
return ErrOverlappingConfigs
|
||||
}
|
||||
|
||||
@@ -244,30 +255,43 @@ func validateNotificationConfig(nConfig notificationConfig) APIErrorCode {
|
||||
// - elasticsearch
|
||||
// - redis
|
||||
// - postgresql
|
||||
// - mysql
|
||||
// - kafka
|
||||
// - webhook
|
||||
func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
|
||||
mSqs = arnSQS{}
|
||||
if !strings.HasPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":") {
|
||||
return mSqs
|
||||
strs := strings.SplitN(queueARN, ":", -1)
|
||||
if len(strs) != 6 {
|
||||
return
|
||||
}
|
||||
sqsType := strings.TrimPrefix(queueARN, minioSqs+serverConfig.GetRegion()+":")
|
||||
switch {
|
||||
case hasSuffix(sqsType, queueTypeAMQP):
|
||||
if serverConfig.GetRegion() != "" {
|
||||
region := strs[3]
|
||||
if region != serverConfig.GetRegion() {
|
||||
return
|
||||
}
|
||||
}
|
||||
sqsType := strs[5]
|
||||
switch sqsType {
|
||||
case queueTypeAMQP:
|
||||
mSqs.Type = queueTypeAMQP
|
||||
case hasSuffix(sqsType, queueTypeNATS):
|
||||
case queueTypeNATS:
|
||||
mSqs.Type = queueTypeNATS
|
||||
case hasSuffix(sqsType, queueTypeElastic):
|
||||
case queueTypeElastic:
|
||||
mSqs.Type = queueTypeElastic
|
||||
case hasSuffix(sqsType, queueTypeRedis):
|
||||
case queueTypeRedis:
|
||||
mSqs.Type = queueTypeRedis
|
||||
case hasSuffix(sqsType, queueTypePostgreSQL):
|
||||
case queueTypePostgreSQL:
|
||||
mSqs.Type = queueTypePostgreSQL
|
||||
case hasSuffix(sqsType, queueTypeKafka):
|
||||
case queueTypeMySQL:
|
||||
mSqs.Type = queueTypeMySQL
|
||||
case queueTypeKafka:
|
||||
mSqs.Type = queueTypeKafka
|
||||
case hasSuffix(sqsType, queueTypeWebhook):
|
||||
case queueTypeWebhook:
|
||||
mSqs.Type = queueTypeWebhook
|
||||
default:
|
||||
errorIf(errors.New("invalid SQS type"), "SQS type: %s", sqsType)
|
||||
} // Add more queues here.
|
||||
mSqs.AccountID = strings.TrimSuffix(sqsType, ":"+mSqs.Type)
|
||||
return mSqs
|
||||
|
||||
mSqs.AccountID = strs[4]
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -259,11 +259,6 @@ func TestQueueARN(t *testing.T) {
|
||||
queueARN: "arn:minio:sns:us-east-1:1:listen",
|
||||
errCode: ErrARNNotification,
|
||||
},
|
||||
// Invalid region 'us-west-1' in queue arn.
|
||||
{
|
||||
queueARN: "arn:minio:sqs:us-west-1:1:redis",
|
||||
errCode: ErrRegionNotification,
|
||||
},
|
||||
// Invalid queue name empty in queue arn.
|
||||
{
|
||||
queueARN: "arn:minio:sqs:us-east-1:1:",
|
||||
@@ -298,6 +293,37 @@ func TestQueueARN(t *testing.T) {
|
||||
t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
|
||||
}
|
||||
}
|
||||
|
||||
// Test when server region is set.
|
||||
rootPath, err = newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
|
||||
testCases = []struct {
|
||||
queueARN string
|
||||
errCode APIErrorCode
|
||||
}{
|
||||
// Incorrect region should produce error.
|
||||
{
|
||||
queueARN: "arn:minio:sqs:us-west-1:1:webhook",
|
||||
errCode: ErrRegionNotification,
|
||||
},
|
||||
// Correct region should not produce error.
|
||||
{
|
||||
queueARN: "arn:minio:sqs:us-east-1:1:webhook",
|
||||
errCode: ErrNone,
|
||||
},
|
||||
}
|
||||
|
||||
// Validate all tests for queue arn.
|
||||
for i, testCase := range testCases {
|
||||
errCode := checkQueueARN(testCase.queueARN)
|
||||
if testCase.errCode != errCode {
|
||||
t.Errorf("Test %d: Expected \"%d\", got \"%d\"", i+1, testCase.errCode, errCode)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test unmarshal queue arn.
|
||||
@@ -337,11 +363,6 @@ func TestUnmarshalSQSARN(t *testing.T) {
|
||||
queueARN: "",
|
||||
Type: "",
|
||||
},
|
||||
// Invalid region 'us-west-1' in queue arn.
|
||||
{
|
||||
queueARN: "arn:minio:sqs:us-west-1:1:redis",
|
||||
Type: "",
|
||||
},
|
||||
// Partial queue arn.
|
||||
{
|
||||
queueARN: "arn:minio:sqs:",
|
||||
@@ -361,4 +382,33 @@ func TestUnmarshalSQSARN(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// Test when the server region is set.
|
||||
rootPath, err = newTestConfig("us-east-1")
|
||||
if err != nil {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
defer removeAll(rootPath)
|
||||
|
||||
testCases = []struct {
|
||||
queueARN string
|
||||
Type string
|
||||
}{
|
||||
// Incorrect region in ARN returns empty mSqs.Type
|
||||
{
|
||||
queueARN: "arn:minio:sqs:us-west-1:1:webhook",
|
||||
Type: "",
|
||||
},
|
||||
// Correct regionin ARN returns valid mSqs.Type
|
||||
{
|
||||
queueARN: "arn:minio:sqs:us-east-1:1:webhook",
|
||||
Type: "webhook",
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
mSqs := unmarshalSqsARN(testCase.queueARN)
|
||||
if testCase.Type != mSqs.Type {
|
||||
t.Errorf("Test %d: Expected \"%s\", got \"%s\"", i+1, testCase.Type, mSqs.Type)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -251,7 +251,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
initBucketPolicies(obj)
|
||||
|
||||
bucketName1 := fmt.Sprintf("%s-1", bucketName)
|
||||
if err := obj.MakeBucket(bucketName1); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName1, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
|
||||
+15
-15
@@ -24,11 +24,6 @@ import (
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// isSSL - returns true with both cert and key exists.
|
||||
func isSSL() bool {
|
||||
return isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())
|
||||
}
|
||||
|
||||
func parsePublicCertFile(certFile string) (certs []*x509.Certificate, err error) {
|
||||
var bytes []byte
|
||||
|
||||
@@ -60,11 +55,6 @@ func parsePublicCertFile(certFile string) (certs []*x509.Certificate, err error)
|
||||
return certs, err
|
||||
}
|
||||
|
||||
// Reads certificate file and returns a list of parsed certificates.
|
||||
func readCertificateChain() ([]*x509.Certificate, error) {
|
||||
return parsePublicCertFile(getPublicCertFile())
|
||||
}
|
||||
|
||||
func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
|
||||
// Get all CA file names.
|
||||
var caFiles []string
|
||||
@@ -100,9 +90,19 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
|
||||
return rootCAs, nil
|
||||
}
|
||||
|
||||
// loadRootCAs fetches CA files provided in minio config and adds them to globalRootCAs
|
||||
// Currently under Windows, there is no way to load system + user CAs at the same time
|
||||
func loadRootCAs() (err error) {
|
||||
globalRootCAs, err = getRootCAs(getCADir())
|
||||
return err
|
||||
func getSSLConfig() (publicCerts []*x509.Certificate, rootCAs *x509.CertPool, secureConn bool, err error) {
|
||||
if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
|
||||
return publicCerts, rootCAs, secureConn, err
|
||||
}
|
||||
|
||||
if publicCerts, err = parsePublicCertFile(getPublicCertFile()); err != nil {
|
||||
return publicCerts, rootCAs, secureConn, err
|
||||
}
|
||||
|
||||
if rootCAs, err = getRootCAs(getCADir()); err != nil {
|
||||
return publicCerts, rootCAs, secureConn, err
|
||||
}
|
||||
|
||||
secureConn = true
|
||||
return publicCerts, rootCAs, secureConn, err
|
||||
}
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net"
|
||||
"os"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
// checkPortAvailability - check if given port is already in use.
|
||||
// Note: The check method tries to listen on given port and closes it.
|
||||
// It is possible to have a disconnected client in this tiny window of time.
|
||||
func checkPortAvailability(port string) error {
|
||||
network := [3]string{"tcp", "tcp4", "tcp6"}
|
||||
for _, n := range network {
|
||||
l, err := net.Listen(n, net.JoinHostPort("", port))
|
||||
if err != nil {
|
||||
if isAddrInUse(err) {
|
||||
// Return error if another process is listening on the
|
||||
// same port.
|
||||
return err
|
||||
}
|
||||
// Ignore any other error (ex. EAFNOSUPPORT)
|
||||
continue
|
||||
}
|
||||
|
||||
// look for error so we don't have dangling connection
|
||||
if err = l.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Return true if err is "address already in use" error.
|
||||
// syscall.EADDRINUSE is available on all OSes.
|
||||
func isAddrInUse(err error) bool {
|
||||
if opErr, ok := err.(*net.OpError); ok {
|
||||
if sysErr, ok := opErr.Err.(*os.SyscallError); ok {
|
||||
if errno, ok := sysErr.Err.(syscall.Errno); ok {
|
||||
if errno == syscall.EADDRINUSE {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,54 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net"
|
||||
"runtime"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests for port availability logic written for server startup sequence.
|
||||
func TestCheckPortAvailability(t *testing.T) {
|
||||
tests := []struct {
|
||||
port string
|
||||
}{
|
||||
{getFreePort()},
|
||||
{getFreePort()},
|
||||
}
|
||||
for _, test := range tests {
|
||||
// This test should pass if the ports are available
|
||||
err := checkPortAvailability(test.port)
|
||||
if err != nil {
|
||||
t.Fatalf("checkPortAvailability test failed for port: %s. Error: %v", test.port, err)
|
||||
}
|
||||
|
||||
// Now use the ports and check again
|
||||
ln, err := net.Listen("tcp", net.JoinHostPort("", test.port))
|
||||
if err != nil {
|
||||
t.Fail()
|
||||
}
|
||||
defer ln.Close()
|
||||
|
||||
err = checkPortAvailability(test.port)
|
||||
|
||||
// Skip if the os is windows due to https://github.com/golang/go/issues/7598
|
||||
if err == nil && runtime.GOOS != globalWindowsOSName {
|
||||
t.Fatalf("checkPortAvailability should fail for port: %s. Error: %v", test.port, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-8
@@ -21,7 +21,6 @@ import (
|
||||
"sync"
|
||||
|
||||
homedir "github.com/minio/go-homedir"
|
||||
"github.com/minio/mc/pkg/console"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -97,9 +96,7 @@ func (config *ConfigDir) GetPrivateKeyFile() string {
|
||||
|
||||
func mustGetDefaultConfigDir() string {
|
||||
homeDir, err := homedir.Dir()
|
||||
if err != nil {
|
||||
console.Fatalln("Unable to get home directory.", err)
|
||||
}
|
||||
fatalIf(err, "Unable to get home directory.")
|
||||
|
||||
return filepath.Join(homeDir, defaultMinioConfigDir)
|
||||
}
|
||||
@@ -133,7 +130,3 @@ func getPublicCertFile() string {
|
||||
func getPrivateKeyFile() string {
|
||||
return configDir.GetPrivateKeyFile()
|
||||
}
|
||||
|
||||
func isConfigFileExists() bool {
|
||||
return isFile(getConfigFile())
|
||||
}
|
||||
|
||||
+666
-303
File diff suppressed because it is too large
Load Diff
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
@@ -45,12 +46,12 @@ func TestServerConfigMigrateV1(t *testing.T) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Check if config v1 is removed from filesystem
|
||||
if _, err := os.Stat(configPath); err == nil || !os.IsNotExist(err) {
|
||||
if _, err := osStat(configPath); err == nil || !os.IsNotExist(err) {
|
||||
t.Fatal("Config V1 file is not purged")
|
||||
}
|
||||
|
||||
// Initialize server config and check again if everything is fine
|
||||
if err := loadConfig(envParams{}); err != nil {
|
||||
if err := loadConfig(); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
}
|
||||
@@ -109,10 +110,23 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
if err := migrateV13ToV14(); err != nil {
|
||||
t.Fatal("migrate v13 to v14 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV14ToV15(); err != nil {
|
||||
t.Fatal("migrate v14 to v15 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV15ToV16(); err != nil {
|
||||
t.Fatal("migrate v15 to v16 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV16ToV17(); err != nil {
|
||||
t.Fatal("migrate v16 to v17 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV17ToV18(); err != nil {
|
||||
t.Fatal("migrate v17 to v18 should succeed when no config file is found")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Test if a config migration from v2 to v12 is successfully done
|
||||
func TestServerConfigMigrateV2toV14(t *testing.T) {
|
||||
// Test if a config migration from v2 to v18 is successfully done
|
||||
func TestServerConfigMigrateV2toV18(t *testing.T) {
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("Init Test config failed")
|
||||
@@ -140,18 +154,19 @@ func TestServerConfigMigrateV2toV14(t *testing.T) {
|
||||
if err := ioutil.WriteFile(configPath, []byte(configJSON), 0644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Fire a migrateConfig()
|
||||
if err := migrateConfig(); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Initialize server config and check again if everything is fine
|
||||
if err := loadConfig(envParams{}); err != nil {
|
||||
if err := loadConfig(); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
|
||||
// Check the version number in the upgraded config file
|
||||
expectedVersion := v14
|
||||
expectedVersion := v18
|
||||
if serverConfig.Version != expectedVersion {
|
||||
t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
|
||||
}
|
||||
@@ -178,7 +193,7 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
configPath := rootPath + "/" + minioConfigFile
|
||||
|
||||
// Create a corrupted config file
|
||||
if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\""), 0644); err != nil {
|
||||
if err := ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
@@ -219,4 +234,52 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
if err := migrateV13ToV14(); err == nil {
|
||||
t.Fatal("migrateConfigV13ToV14() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV14ToV15(); err == nil {
|
||||
t.Fatal("migrateConfigV14ToV15() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV15ToV16(); err == nil {
|
||||
t.Fatal("migrateConfigV15ToV16() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV16ToV17(); err == nil {
|
||||
t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV17ToV18(); err == nil {
|
||||
t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("Init Test config failed")
|
||||
}
|
||||
// remove the root directory after the test ends.
|
||||
defer removeAll(rootPath)
|
||||
|
||||
setConfigDir(rootPath)
|
||||
configPath := rootPath + "/" + minioConfigFile
|
||||
|
||||
for i := 3; i <= 17; i++ {
|
||||
// Create a corrupted config file
|
||||
if err = ioutil.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
|
||||
0644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Test different migrate versions and be sure they are returning an error
|
||||
if err = migrateConfig(); err == nil {
|
||||
t.Fatal("migrateConfig() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
// Create a corrupted config file for version '2'.
|
||||
if err = ioutil.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Test different migrate versions and be sure they are returning an error
|
||||
if err = migrateConfig(); err == nil {
|
||||
t.Fatal("migrateConfig() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
+90
-151
@@ -16,30 +16,7 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
)
|
||||
|
||||
func loadOldConfig(configFile string, config interface{}) (interface{}, error) {
|
||||
if _, err := os.Stat(configFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
qc, err := quick.New(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = qc.Load(configFile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
import "sync"
|
||||
|
||||
/////////////////// Config V1 ///////////////////
|
||||
type configV1 struct {
|
||||
@@ -48,16 +25,6 @@ type configV1 struct {
|
||||
SecretKey string `json:"secretAccessKey"`
|
||||
}
|
||||
|
||||
// loadConfigV1 load config
|
||||
func loadConfigV1() (*configV1, error) {
|
||||
configFile := filepath.Join(getConfigDir(), "fsUsers.json")
|
||||
config, err := loadOldConfig(configFile, &configV1{Version: "1"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*configV1), err
|
||||
}
|
||||
|
||||
/////////////////// Config V2 ///////////////////
|
||||
type configV2 struct {
|
||||
Version string `json:"version"`
|
||||
@@ -80,18 +47,7 @@ type configV2 struct {
|
||||
} `json:"fileLogger"`
|
||||
}
|
||||
|
||||
// loadConfigV2 load config version '2'.
|
||||
func loadConfigV2() (*configV2, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &configV2{Version: "2"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*configV2), err
|
||||
}
|
||||
|
||||
/////////////////// Config V3 ///////////////////
|
||||
|
||||
// backendV3 type.
|
||||
type backendV3 struct {
|
||||
Type string `json:"type"`
|
||||
@@ -143,16 +99,6 @@ type configV3 struct {
|
||||
Logger loggerV3 `json:"logger"`
|
||||
}
|
||||
|
||||
// loadConfigV3 load config version '3'.
|
||||
func loadConfigV3() (*configV3, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &configV3{Version: "3"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*configV3), err
|
||||
}
|
||||
|
||||
// logger type representing version '4' logger config.
|
||||
type loggerV4 struct {
|
||||
Console struct {
|
||||
@@ -183,16 +129,6 @@ type configV4 struct {
|
||||
Logger loggerV4 `json:"logger"`
|
||||
}
|
||||
|
||||
// loadConfigV4 load config version '4'.
|
||||
func loadConfigV4() (*configV4, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &configV4{Version: "4"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*configV4), err
|
||||
}
|
||||
|
||||
// logger type representing version '5' logger config.
|
||||
type loggerV5 struct {
|
||||
Console struct {
|
||||
@@ -250,20 +186,22 @@ type configV5 struct {
|
||||
Logger loggerV5 `json:"logger"`
|
||||
}
|
||||
|
||||
// loadConfigV5 load config version '5'.
|
||||
func loadConfigV5() (*configV5, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &configV5{Version: "5"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*configV5), err
|
||||
// consoleLogger - default logger if not other logging is enabled.
|
||||
type consoleLoggerV1 struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
type fileLoggerV1 struct {
|
||||
Enable bool `json:"enable"`
|
||||
Filename string `json:"fileName"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
type loggerV6 struct {
|
||||
Console consoleLogger `json:"console"`
|
||||
File fileLogger `json:"file"`
|
||||
Syslog syslogLoggerV3 `json:"syslog"`
|
||||
Console consoleLoggerV1 `json:"console"`
|
||||
File fileLoggerV1 `json:"file"`
|
||||
Syslog syslogLoggerV3 `json:"syslog"`
|
||||
}
|
||||
|
||||
// configV6 server configuration version '6'.
|
||||
@@ -281,16 +219,6 @@ type configV6 struct {
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
// loadConfigV6 load config version '6'.
|
||||
func loadConfigV6() (*configV6, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &configV6{Version: "6"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*configV6), err
|
||||
}
|
||||
|
||||
// Notifier represents collection of supported notification queues in version
|
||||
// 1 without NATS streaming.
|
||||
type notifierV1 struct {
|
||||
@@ -331,16 +259,6 @@ type serverConfigV7 struct {
|
||||
rwMutex *sync.RWMutex
|
||||
}
|
||||
|
||||
// loadConfigV7 load config version '7'.
|
||||
func loadConfigV7() (*serverConfigV7, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &serverConfigV7{Version: "7"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*serverConfigV7), err
|
||||
}
|
||||
|
||||
// serverConfigV8 server configuration version '8'. Adds NATS notifier
|
||||
// configuration.
|
||||
type serverConfigV8 struct {
|
||||
@@ -360,16 +278,6 @@ type serverConfigV8 struct {
|
||||
rwMutex *sync.RWMutex
|
||||
}
|
||||
|
||||
// loadConfigV8 load config version '8'.
|
||||
func loadConfigV8() (*serverConfigV8, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &serverConfigV8{Version: "8"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*serverConfigV8), err
|
||||
}
|
||||
|
||||
// serverConfigV9 server configuration version '9'. Adds PostgreSQL
|
||||
// notifier configuration.
|
||||
type serverConfigV9 struct {
|
||||
@@ -389,13 +297,10 @@ type serverConfigV9 struct {
|
||||
rwMutex *sync.RWMutex
|
||||
}
|
||||
|
||||
func loadConfigV9() (*serverConfigV9, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &serverConfigV9{Version: "9"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*serverConfigV9), err
|
||||
type loggerV7 struct {
|
||||
sync.RWMutex
|
||||
Console consoleLoggerV1 `json:"console"`
|
||||
File fileLoggerV1 `json:"file"`
|
||||
}
|
||||
|
||||
// serverConfigV10 server configuration version '10' which is like
|
||||
@@ -409,21 +314,12 @@ type serverConfigV10 struct {
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger logger `json:"logger"`
|
||||
Logger loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
func loadConfigV10() (*serverConfigV10, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &serverConfigV10{Version: "10"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*serverConfigV10), err
|
||||
}
|
||||
|
||||
// natsNotifyV1 - structure was valid until config V 11
|
||||
type natsNotifyV1 struct {
|
||||
Enable bool `json:"enable"`
|
||||
@@ -446,21 +342,12 @@ type serverConfigV11 struct {
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger logger `json:"logger"`
|
||||
Logger loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
func loadConfigV11() (*serverConfigV11, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &serverConfigV11{Version: "11"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*serverConfigV11), err
|
||||
}
|
||||
|
||||
// serverConfigV12 server configuration version '12' which is like
|
||||
// version '11' except it adds support for NATS streaming notifications.
|
||||
type serverConfigV12 struct {
|
||||
@@ -471,21 +358,12 @@ type serverConfigV12 struct {
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger logger `json:"logger"`
|
||||
Logger loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV2 `json:"notify"`
|
||||
}
|
||||
|
||||
func loadConfigV12() (*serverConfigV12, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &serverConfigV12{Version: "12"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*serverConfigV12), err
|
||||
}
|
||||
|
||||
// serverConfigV13 server configuration version '13' which is like
|
||||
// version '12' except it adds support for webhook notification.
|
||||
type serverConfigV13 struct {
|
||||
@@ -496,17 +374,78 @@ type serverConfigV13 struct {
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *logger `json:"logger"`
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
func loadConfigV13() (*serverConfigV13, error) {
|
||||
configFile := getConfigFile()
|
||||
config, err := loadOldConfig(configFile, &serverConfigV13{Version: "13"})
|
||||
if config == nil {
|
||||
return nil, err
|
||||
}
|
||||
return config.(*serverConfigV13), err
|
||||
// serverConfigV14 server configuration version '14' which is like
|
||||
// version '13' except it adds support of browser param.
|
||||
type serverConfigV14 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV15 server configuration version '15' which is like
|
||||
// version '14' except it adds mysql support
|
||||
type serverConfigV15 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV16 server configuration version '16' which is like
|
||||
// version '15' except it makes a change to logging configuration.
|
||||
type serverConfigV16 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV17 server configuration version '17' which is like
|
||||
// version '16' except it adds support for "format" parameter in
|
||||
// database event notification targets: PostgreSQL, MySQL, Redis and
|
||||
// Elasticsearch.
|
||||
type serverConfigV17 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
@@ -20,51 +20,119 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/quick"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// Read Write mutex for safe access to ServerConfig.
|
||||
var serverConfigMu sync.RWMutex
|
||||
|
||||
// Config version
|
||||
var v14 = "14"
|
||||
const v18 = "18"
|
||||
|
||||
// serverConfigV14 server configuration version '14' which is like
|
||||
// version '13' except it adds support of browser param.
|
||||
type serverConfigV14 struct {
|
||||
var (
|
||||
// serverConfig server config.
|
||||
serverConfig *serverConfigV18
|
||||
serverConfigMu sync.RWMutex
|
||||
)
|
||||
|
||||
// serverConfigV18 server configuration version '18' which is like
|
||||
// version '17' except it adds support for "deliveryMode" parameter in
|
||||
// the AMQP notification target.
|
||||
type serverConfigV18 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser string `json:"browser"`
|
||||
Credential credential `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *logger `json:"logger"`
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
func newServerConfigV14() *serverConfigV14 {
|
||||
srvCfg := &serverConfigV14{
|
||||
Version: v14,
|
||||
Region: globalMinioDefaultRegion,
|
||||
Logger: &logger{},
|
||||
Notify: ¬ifier{},
|
||||
// GetVersion get current config version.
|
||||
func (s *serverConfigV18) GetVersion() string {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
return s.Version
|
||||
}
|
||||
|
||||
// SetRegion set new region.
|
||||
func (s *serverConfigV18) SetRegion(region string) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
s.Region = region
|
||||
}
|
||||
|
||||
// GetRegion get current region.
|
||||
func (s *serverConfigV18) GetRegion() string {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
return s.Region
|
||||
}
|
||||
|
||||
// SetCredentials set new credentials.
|
||||
func (s *serverConfigV18) SetCredential(creds credential) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
// Set updated credential.
|
||||
s.Credential = creds
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s *serverConfigV18) GetCredential() credential {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
return s.Credential
|
||||
}
|
||||
|
||||
// SetBrowser set if browser is enabled.
|
||||
func (s *serverConfigV18) SetBrowser(b bool) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
// Set the new value.
|
||||
s.Browser = BrowserFlag(b)
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s *serverConfigV18) GetBrowser() bool {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
return bool(s.Browser)
|
||||
}
|
||||
|
||||
// Save config.
|
||||
func (s *serverConfigV18) Save() error {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
// Save config file.
|
||||
return quick.Save(getConfigFile(), s)
|
||||
}
|
||||
|
||||
func newServerConfigV18() *serverConfigV18 {
|
||||
srvCfg := &serverConfigV18{
|
||||
Version: v18,
|
||||
Credential: mustGetNewCredential(),
|
||||
Region: globalMinioDefaultRegion,
|
||||
Browser: true,
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
}
|
||||
srvCfg.SetCredential(mustGetNewCredential())
|
||||
srvCfg.SetBrowser("on")
|
||||
|
||||
// Enable console logger by default on a fresh run.
|
||||
srvCfg.Logger.Console = consoleLogger{
|
||||
Enable: true,
|
||||
Level: "error",
|
||||
}
|
||||
srvCfg.Logger.Console = NewConsoleLogger()
|
||||
|
||||
// Make sure to initialize notification configs.
|
||||
srvCfg.Notify.AMQP = make(map[string]amqpNotify)
|
||||
@@ -77,6 +145,8 @@ func newServerConfigV14() *serverConfigV14 {
|
||||
srvCfg.Notify.NATS["1"] = natsNotify{}
|
||||
srvCfg.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
|
||||
srvCfg.Notify.PostgreSQL["1"] = postgreSQLNotify{}
|
||||
srvCfg.Notify.MySQL = make(map[string]mySQLNotify)
|
||||
srvCfg.Notify.MySQL["1"] = mySQLNotify{}
|
||||
srvCfg.Notify.Kafka = make(map[string]kafkaNotify)
|
||||
srvCfg.Notify.Kafka["1"] = kafkaNotify{}
|
||||
srvCfg.Notify.Webhook = make(map[string]webhookNotify)
|
||||
@@ -87,22 +157,21 @@ func newServerConfigV14() *serverConfigV14 {
|
||||
|
||||
// newConfig - initialize a new server config, saves env parameters if
|
||||
// found, otherwise use default parameters
|
||||
func newConfig(envParams envParams) error {
|
||||
func newConfig() error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfigV14()
|
||||
srvCfg := newServerConfigV18()
|
||||
|
||||
// If env is set for a fresh start, save them to config file.
|
||||
// If env is set override the credentials from config file.
|
||||
if globalIsEnvCreds {
|
||||
srvCfg.SetCredential(envParams.creds)
|
||||
srvCfg.SetCredential(globalActiveCred)
|
||||
}
|
||||
|
||||
if globalIsEnvBrowser {
|
||||
srvCfg.SetBrowser(envParams.browser)
|
||||
srvCfg.SetBrowser(globalIsBrowserEnabled)
|
||||
}
|
||||
|
||||
// Create config path.
|
||||
if err := createConfigDir(); err != nil {
|
||||
return err
|
||||
if globalIsEnvRegion {
|
||||
srvCfg.SetRegion(globalServerRegion)
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
@@ -116,51 +185,6 @@ func newConfig(envParams envParams) error {
|
||||
return serverConfig.Save()
|
||||
}
|
||||
|
||||
// loadConfig - loads a new config from disk, overrides params from env
|
||||
// if found and valid
|
||||
func loadConfig(envParams envParams) error {
|
||||
configFile := getConfigFile()
|
||||
if _, err := os.Stat(configFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
srvCfg := &serverConfigV14{}
|
||||
|
||||
qc, err := quick.New(srvCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = qc.Load(configFile); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// If env is set override the credentials from config file.
|
||||
if globalIsEnvCreds {
|
||||
srvCfg.SetCredential(envParams.creds)
|
||||
}
|
||||
|
||||
if globalIsEnvBrowser {
|
||||
srvCfg.SetBrowser(envParams.browser)
|
||||
}
|
||||
|
||||
if strings.ToLower(srvCfg.GetBrowser()) == "off" {
|
||||
globalIsBrowserEnabled = false
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
serverConfigMu.Lock()
|
||||
// Save the loaded config globally.
|
||||
serverConfig = srvCfg
|
||||
serverConfigMu.Unlock()
|
||||
|
||||
if serverConfig.Version != v14 {
|
||||
return errors.New("Unsupported config version `" + serverConfig.Version + "`.")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// doCheckDupJSONKeys recursively detects duplicate json keys
|
||||
func doCheckDupJSONKeys(key, value gjson.Result) error {
|
||||
// Key occurrences map of the current scope to count
|
||||
@@ -212,159 +236,86 @@ func checkDupJSONKeys(json string) error {
|
||||
return doCheckDupJSONKeys(rootKey, config)
|
||||
}
|
||||
|
||||
// validateConfig checks for
|
||||
func validateConfig() error {
|
||||
// getValidConfig - returns valid server configuration
|
||||
func getValidConfig() (*serverConfigV18, error) {
|
||||
srvCfg := &serverConfigV18{
|
||||
Region: globalMinioDefaultRegion,
|
||||
Browser: true,
|
||||
}
|
||||
|
||||
// Get file config path
|
||||
configFile := getConfigFile()
|
||||
|
||||
srvCfg := &serverConfigV14{}
|
||||
|
||||
// Load config file
|
||||
qc, err := quick.New(srvCfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err = qc.Load(configFile); err != nil {
|
||||
return err
|
||||
if _, err := quick.Load(configFile, srvCfg); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check if config version is valid
|
||||
if srvCfg.GetVersion() != v14 {
|
||||
return errors.New("bad config version, expected: " + v14)
|
||||
if srvCfg.Version != v18 {
|
||||
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v18, srvCfg.Version)
|
||||
}
|
||||
|
||||
// Load config file json and check for duplication json keys
|
||||
jsonBytes, err := ioutil.ReadFile(configFile)
|
||||
if err != nil {
|
||||
return err
|
||||
return nil, err
|
||||
}
|
||||
if err := checkDupJSONKeys(string(jsonBytes)); err != nil {
|
||||
return err
|
||||
if err = checkDupJSONKeys(string(jsonBytes)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate region field
|
||||
if srvCfg.GetRegion() == "" {
|
||||
return errors.New("Region config value cannot be empty")
|
||||
}
|
||||
// Validate credential fields only when
|
||||
// they are not set via the environment
|
||||
|
||||
// Validate browser field
|
||||
if b := strings.ToLower(srvCfg.GetBrowser()); b != "on" && b != "off" {
|
||||
return fmt.Errorf("Browser config value %s is invalid", b)
|
||||
}
|
||||
|
||||
// Validate credential field
|
||||
if !srvCfg.Credential.IsValid() {
|
||||
return errors.New("invalid credential")
|
||||
// Error out if global is env credential is not set and config has invalid credential
|
||||
if !globalIsEnvCreds && !srvCfg.Credential.IsValid() {
|
||||
return nil, errors.New("invalid credential in config file " + configFile)
|
||||
}
|
||||
|
||||
// Validate logger field
|
||||
if err := srvCfg.Logger.Validate(); err != nil {
|
||||
return err
|
||||
if err = srvCfg.Logger.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate notify field
|
||||
if err := srvCfg.Notify.Validate(); err != nil {
|
||||
return err
|
||||
if err = srvCfg.Notify.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil
|
||||
return srvCfg, nil
|
||||
}
|
||||
|
||||
// serverConfig server config.
|
||||
var serverConfig *serverConfigV14
|
||||
|
||||
// GetVersion get current config version.
|
||||
func (s serverConfigV14) GetVersion() string {
|
||||
serverConfigMu.RLock()
|
||||
defer serverConfigMu.RUnlock()
|
||||
|
||||
return s.Version
|
||||
}
|
||||
|
||||
// SetRegion set new region.
|
||||
func (s *serverConfigV14) SetRegion(region string) {
|
||||
serverConfigMu.Lock()
|
||||
defer serverConfigMu.Unlock()
|
||||
|
||||
// Empty region means "us-east-1" by default from S3 spec.
|
||||
if region == "" {
|
||||
region = "us-east-1"
|
||||
}
|
||||
s.Region = region
|
||||
}
|
||||
|
||||
// GetRegion get current region.
|
||||
func (s serverConfigV14) GetRegion() string {
|
||||
serverConfigMu.RLock()
|
||||
defer serverConfigMu.RUnlock()
|
||||
|
||||
if s.Region != "" {
|
||||
return s.Region
|
||||
} // region empty
|
||||
|
||||
// Empty region means "us-east-1" by default from S3 spec.
|
||||
return "us-east-1"
|
||||
}
|
||||
|
||||
// SetCredentials set new credentials.
|
||||
func (s *serverConfigV14) SetCredential(creds credential) {
|
||||
serverConfigMu.Lock()
|
||||
defer serverConfigMu.Unlock()
|
||||
|
||||
// Set updated credential.
|
||||
s.Credential = creds
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s serverConfigV14) GetCredential() credential {
|
||||
serverConfigMu.RLock()
|
||||
defer serverConfigMu.RUnlock()
|
||||
|
||||
return s.Credential
|
||||
}
|
||||
|
||||
// SetBrowser set if browser is enabled.
|
||||
func (s *serverConfigV14) SetBrowser(v string) {
|
||||
serverConfigMu.Lock()
|
||||
defer serverConfigMu.Unlock()
|
||||
|
||||
// Set browser param
|
||||
if v == "" {
|
||||
v = "on" // Browser is on by default.
|
||||
}
|
||||
|
||||
// Set the new value.
|
||||
s.Browser = v
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s serverConfigV14) GetBrowser() string {
|
||||
serverConfigMu.RLock()
|
||||
defer serverConfigMu.RUnlock()
|
||||
|
||||
if s.Browser != "" {
|
||||
return s.Browser
|
||||
} // empty browser.
|
||||
|
||||
// Empty browser means "on" by default.
|
||||
return "on"
|
||||
}
|
||||
|
||||
// Save config.
|
||||
func (s serverConfigV14) Save() error {
|
||||
serverConfigMu.RLock()
|
||||
defer serverConfigMu.RUnlock()
|
||||
|
||||
// get config file.
|
||||
configFile := getConfigFile()
|
||||
|
||||
// initialize quick.
|
||||
qc, err := quick.New(&s)
|
||||
// loadConfig - loads a new config from disk, overrides params from env
|
||||
// if found and valid
|
||||
func loadConfig() error {
|
||||
srvCfg, err := getValidConfig()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Save config file.
|
||||
return qc.Save(configFile)
|
||||
// If env is set override the credentials from config file.
|
||||
if globalIsEnvCreds {
|
||||
srvCfg.SetCredential(globalActiveCred)
|
||||
}
|
||||
|
||||
if globalIsEnvBrowser {
|
||||
srvCfg.SetBrowser(globalIsBrowserEnabled)
|
||||
}
|
||||
|
||||
if globalIsEnvRegion {
|
||||
srvCfg.SetRegion(globalServerRegion)
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
serverConfigMu.Lock()
|
||||
serverConfig = srvCfg
|
||||
if !globalIsEnvCreds {
|
||||
globalActiveCred = serverConfig.GetCredential()
|
||||
}
|
||||
if !globalIsEnvBrowser {
|
||||
globalIsBrowserEnabled = serverConfig.GetBrowser()
|
||||
}
|
||||
if !globalIsEnvRegion {
|
||||
globalServerRegion = serverConfig.GetRegion()
|
||||
}
|
||||
serverConfigMu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -76,38 +76,41 @@ func TestServerConfig(t *testing.T) {
|
||||
serverConfig.Notify.SetWebhookByID("2", webhookNotify{})
|
||||
savedNotifyCfg5 := serverConfig.Notify.GetWebhookByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg5, webhookNotify{}) {
|
||||
t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg3)
|
||||
t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg5)
|
||||
}
|
||||
|
||||
// Set new console logger.
|
||||
serverConfig.Logger.SetConsole(consoleLogger{
|
||||
Enable: true,
|
||||
})
|
||||
// Set new Webhook notification id.
|
||||
serverConfig.Notify.SetMySQLByID("2", mySQLNotify{})
|
||||
savedNotifyCfg6 := serverConfig.Notify.GetMySQLByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg6, mySQLNotify{}) {
|
||||
t.Errorf("Expecting Webhook config %#v found %#v", mySQLNotify{}, savedNotifyCfg6)
|
||||
}
|
||||
|
||||
consoleLogger := NewConsoleLogger()
|
||||
serverConfig.Logger.SetConsole(consoleLogger)
|
||||
consoleCfg := serverConfig.Logger.GetConsole()
|
||||
if !reflect.DeepEqual(consoleCfg, consoleLogger{Enable: true}) {
|
||||
t.Errorf("Expecting console logger config %#v found %#v", consoleLogger{Enable: true}, consoleCfg)
|
||||
if !reflect.DeepEqual(consoleCfg, consoleLogger) {
|
||||
t.Errorf("Expecting console logger config %#v found %#v", consoleLogger, consoleCfg)
|
||||
}
|
||||
// Set new console logger.
|
||||
serverConfig.Logger.SetConsole(consoleLogger{
|
||||
Enable: false,
|
||||
})
|
||||
consoleLogger.Enable = false
|
||||
serverConfig.Logger.SetConsole(consoleLogger)
|
||||
|
||||
// Set new file logger.
|
||||
serverConfig.Logger.SetFile(fileLogger{
|
||||
Enable: true,
|
||||
})
|
||||
fileLogger := NewFileLogger("test-log-file")
|
||||
serverConfig.Logger.SetFile(fileLogger)
|
||||
fileCfg := serverConfig.Logger.GetFile()
|
||||
if !reflect.DeepEqual(fileCfg, fileLogger{Enable: true}) {
|
||||
t.Errorf("Expecting file logger config %#v found %#v", fileLogger{Enable: true}, consoleCfg)
|
||||
if !reflect.DeepEqual(fileCfg, fileLogger) {
|
||||
t.Errorf("Expecting file logger config %#v found %#v", fileLogger, fileCfg)
|
||||
}
|
||||
// Set new file logger.
|
||||
serverConfig.Logger.SetFile(fileLogger{
|
||||
Enable: false,
|
||||
})
|
||||
fileLogger.Enable = false
|
||||
serverConfig.Logger.SetFile(fileLogger)
|
||||
|
||||
// Match version.
|
||||
if serverConfig.GetVersion() != v14 {
|
||||
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v14)
|
||||
if serverConfig.GetVersion() != v18 {
|
||||
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v18)
|
||||
}
|
||||
|
||||
// Attempt to save.
|
||||
@@ -119,7 +122,7 @@ func TestServerConfig(t *testing.T) {
|
||||
setConfigDir(rootPath)
|
||||
|
||||
// Initialize server config.
|
||||
if err := loadConfig(envParams{}); err != nil {
|
||||
if err := loadConfig(); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
}
|
||||
@@ -135,10 +138,10 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
os.Setenv("MINIO_SECRET_KEY", "minio123")
|
||||
defer os.Unsetenv("MINIO_SECRET_KEY")
|
||||
|
||||
defer func() {
|
||||
globalIsEnvBrowser = false
|
||||
globalIsEnvCreds = false
|
||||
}()
|
||||
os.Setenv("MINIO_REGION", "us-west-1")
|
||||
defer os.Unsetenv("MINIO_REGION")
|
||||
|
||||
defer resetGlobalIsEnvs()
|
||||
|
||||
// Get test root.
|
||||
rootPath, err := getTestRoot()
|
||||
@@ -146,6 +149,8 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
serverHandleEnvVars()
|
||||
|
||||
// Do this only once here.
|
||||
setConfigDir(rootPath)
|
||||
|
||||
@@ -156,8 +161,13 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
defer removeAll(rootPath)
|
||||
|
||||
// Check if serverConfig has
|
||||
if serverConfig.GetBrowser() != "off" {
|
||||
t.Errorf("Expecting browser `off` found %s", serverConfig.GetBrowser())
|
||||
if serverConfig.GetBrowser() {
|
||||
t.Errorf("Expecting browser is set to false found %v", serverConfig.GetBrowser())
|
||||
}
|
||||
|
||||
// Check if serverConfig has
|
||||
if serverConfig.GetRegion() != "us-west-1" {
|
||||
t.Errorf("Expecting region to be \"us-west-1\" found %v", serverConfig.GetRegion())
|
||||
}
|
||||
|
||||
// Check if serverConfig has
|
||||
@@ -170,6 +180,7 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
if cred.SecretKey != "minio123" {
|
||||
t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestCheckDupJSONKeys(t *testing.T) {
|
||||
@@ -212,7 +223,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
|
||||
configPath := filepath.Join(rootPath, minioConfigFile)
|
||||
|
||||
v := v14
|
||||
v := v18
|
||||
|
||||
testCases := []struct {
|
||||
configData string
|
||||
@@ -248,39 +259,63 @@ func TestValidateConfig(t *testing.T) {
|
||||
// Test 10 - duplicated json keys
|
||||
{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
|
||||
|
||||
// Test 11 - Wrong Console logger level
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "console": { "enable": true, "level": "foo" } }}`, false},
|
||||
// Test 11 - empty filename field in File
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "filename": "" } }}`, false},
|
||||
|
||||
// Test 12 - Wrong File logger level
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "level": "foo" } }}`, false},
|
||||
|
||||
// Test 13 - Test AMQP
|
||||
// Test 12 - Test AMQP
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
|
||||
|
||||
// Test 14 - Test NATS
|
||||
// Test 13 - Test NATS
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
|
||||
// Test 15 - Test ElasticSearch
|
||||
// Test 14 - Test ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
|
||||
|
||||
// Test 16 - Test Redis
|
||||
// Test 15 - Test Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
|
||||
|
||||
// Test 17 - Test PostgreSQL
|
||||
// Test 16 - Test PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
// Test 18 - Test Kafka
|
||||
// Test 17 - Test Kafka
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
|
||||
|
||||
// Test 19 - Test Webhook
|
||||
// Test 18 - Test Webhook
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
|
||||
|
||||
// Test 20 - Test MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
// Test 21 - Test Format for MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
|
||||
|
||||
// Test 22 - Test valid Format for MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
|
||||
// Test 23 - Test Format for PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
|
||||
|
||||
// Test 24 - Test valid Format for PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
|
||||
// Test 25 - Test Format for ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
|
||||
|
||||
// Test 26 - Test valid Format for ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
|
||||
|
||||
// Test 27 - Test Format for Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
|
||||
|
||||
// Test 28 - Test valid Format for Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
if werr := ioutil.WriteFile(configPath, []byte(testCase.configData), 0700); werr != nil {
|
||||
t.Fatal(werr)
|
||||
}
|
||||
verr := validateConfig()
|
||||
_, verr := getValidConfig()
|
||||
if testCase.shouldPass && verr != nil {
|
||||
t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ConsoleLogger - console logger which logs into stderr.
|
||||
type ConsoleLogger struct {
|
||||
BaseLogTarget
|
||||
}
|
||||
|
||||
// Fire - log entry handler.
|
||||
func (logger ConsoleLogger) Fire(entry *logrus.Entry) error {
|
||||
if !logger.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
msgBytes, err := logger.formatter.Format(entry)
|
||||
if err == nil {
|
||||
fmt.Fprintf(os.Stderr, string(msgBytes))
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// String - represents ConsoleLogger as string.
|
||||
func (logger ConsoleLogger) String() string {
|
||||
enableStr := "disabled"
|
||||
if logger.Enable {
|
||||
enableStr = "enabled"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("console:%s", enableStr)
|
||||
}
|
||||
|
||||
// NewConsoleLogger - return new console logger object.
|
||||
func NewConsoleLogger() (logger ConsoleLogger) {
|
||||
logger.Enable = true
|
||||
logger.formatter = new(logrus.TextFormatter)
|
||||
|
||||
return logger
|
||||
}
|
||||
|
||||
// InitConsoleLogger - initializes console logger.
|
||||
func InitConsoleLogger(logger *ConsoleLogger) {
|
||||
if !logger.Enable {
|
||||
return
|
||||
}
|
||||
|
||||
if logger.formatter == nil {
|
||||
logger.formatter = new(logrus.TextFormatter)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
+31
-43
@@ -21,47 +21,39 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
|
||||
"github.com/minio/mc/pkg/console"
|
||||
|
||||
"golang.org/x/crypto/bcrypt"
|
||||
)
|
||||
|
||||
const (
|
||||
accessKeyMinLen = 5
|
||||
accessKeyMaxLen = 20
|
||||
secretKeyMinLen = 8
|
||||
secretKeyMaxLenAmazon = 40
|
||||
alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
alphaNumericTableLen = byte(len(alphaNumericTable))
|
||||
// Minimum length for Minio access key.
|
||||
accessKeyMinLen = 5
|
||||
|
||||
// Maximum length for Minio access key.
|
||||
accessKeyMaxLen = 20
|
||||
|
||||
// Minimum length for Minio secret key for both server and gateway mode.
|
||||
secretKeyMinLen = 8
|
||||
|
||||
// Maximum secret key length for Minio, this
|
||||
// is used when autogenerating new credentials.
|
||||
secretKeyMaxLenMinio = 40
|
||||
|
||||
// Maximum secret key length allowed from client side
|
||||
// caters for both server and gateway mode.
|
||||
secretKeyMaxLen = 100
|
||||
|
||||
// Alpha numeric table used for generating access keys.
|
||||
alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||
|
||||
// Total length of the alpha numeric table.
|
||||
alphaNumericTableLen = byte(len(alphaNumericTable))
|
||||
)
|
||||
|
||||
// Common errors generated for access and secret key validation.
|
||||
var (
|
||||
errInvalidAccessKeyLength = errors.New("Invalid access key, access key should be 5 to 20 characters in length")
|
||||
errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 40 characters in length")
|
||||
errInvalidSecretKeyLength = errors.New("Invalid secret key, secret key should be 8 to 100 characters in length")
|
||||
)
|
||||
var secretKeyMaxLen = secretKeyMaxLenAmazon
|
||||
|
||||
func mustGetAccessKey() string {
|
||||
keyBytes := make([]byte, accessKeyMaxLen)
|
||||
if _, err := rand.Read(keyBytes); err != nil {
|
||||
console.Fatalf("Unable to generate access key. Err: %s.\n", err)
|
||||
}
|
||||
|
||||
for i := 0; i < accessKeyMaxLen; i++ {
|
||||
keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
|
||||
}
|
||||
|
||||
return string(keyBytes)
|
||||
}
|
||||
|
||||
func mustGetSecretKey() string {
|
||||
keyBytes := make([]byte, secretKeyMaxLen)
|
||||
if _, err := rand.Read(keyBytes); err != nil {
|
||||
console.Fatalf("Unable to generate secret key. Err: %s.\n", err)
|
||||
}
|
||||
|
||||
return string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
|
||||
}
|
||||
|
||||
// isAccessKeyValid - validate access key for right length.
|
||||
func isAccessKeyValid(accessKey string) bool {
|
||||
@@ -127,25 +119,21 @@ func createCredential(accessKey, secretKey string) (cred credential, err error)
|
||||
func mustGetNewCredential() credential {
|
||||
// Generate access key.
|
||||
keyBytes := make([]byte, accessKeyMaxLen)
|
||||
if _, err := rand.Read(keyBytes); err != nil {
|
||||
console.Fatalln("Unable to generate access key.", err)
|
||||
}
|
||||
_, err := rand.Read(keyBytes)
|
||||
fatalIf(err, "Unable to generate access key.")
|
||||
for i := 0; i < accessKeyMaxLen; i++ {
|
||||
keyBytes[i] = alphaNumericTable[keyBytes[i]%alphaNumericTableLen]
|
||||
}
|
||||
accessKey := string(keyBytes)
|
||||
|
||||
// Generate secret key.
|
||||
keyBytes = make([]byte, secretKeyMaxLen)
|
||||
if _, err := rand.Read(keyBytes); err != nil {
|
||||
console.Fatalln("Unable to generate secret key.", err)
|
||||
}
|
||||
secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLen])
|
||||
keyBytes = make([]byte, secretKeyMaxLenMinio)
|
||||
_, err = rand.Read(keyBytes)
|
||||
fatalIf(err, "Unable to generate secret key.")
|
||||
secretKey := string([]byte(base64.StdEncoding.EncodeToString(keyBytes))[:secretKeyMaxLenMinio])
|
||||
|
||||
cred, err := createCredential(accessKey, secretKey)
|
||||
if err != nil {
|
||||
console.Fatalln("Unable to generate new credential.", err)
|
||||
}
|
||||
fatalIf(err, "Unable to generate new credential.")
|
||||
|
||||
return cred
|
||||
}
|
||||
|
||||
@@ -23,6 +23,9 @@ func TestMustGetNewCredential(t *testing.T) {
|
||||
if !cred.IsValid() {
|
||||
t.Fatalf("Failed to get new valid credential")
|
||||
}
|
||||
if len(cred.SecretKey) != secretKeyMaxLenMinio {
|
||||
t.Fatalf("Invalid length %d of the secretKey credential generated, expected %d", len(cred.SecretKey), secretKeyMaxLenMinio)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateCredential(t *testing.T) {
|
||||
@@ -42,7 +45,7 @@ func TestCreateCredential(t *testing.T) {
|
||||
// Secret key too small.
|
||||
{"myuser", "pass", false, errInvalidSecretKeyLength},
|
||||
// Secret key too long.
|
||||
{"myuser", "pass1234567890123456789012345678901234567", false, errInvalidSecretKeyLength},
|
||||
{"myuser", "pass1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", false, errInvalidSecretKeyLength},
|
||||
// Success when access key contains leading/trailing spaces.
|
||||
{" user ", cred.SecretKey, true, nil},
|
||||
{"myuser", "mypassword", true, nil},
|
||||
|
||||
+408
@@ -0,0 +1,408 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"path"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
)
|
||||
|
||||
// EndpointType - enum for endpoint type.
|
||||
type EndpointType int
|
||||
|
||||
const (
|
||||
// PathEndpointType - path style endpoint type enum.
|
||||
PathEndpointType EndpointType = iota + 1
|
||||
|
||||
// URLEndpointType - URL style endpoint type enum.
|
||||
URLEndpointType
|
||||
)
|
||||
|
||||
// Endpoint - any type of endpoint.
|
||||
type Endpoint struct {
|
||||
*url.URL
|
||||
IsLocal bool
|
||||
}
|
||||
|
||||
func (endpoint Endpoint) String() string {
|
||||
if endpoint.Host == "" {
|
||||
return endpoint.Path
|
||||
}
|
||||
|
||||
return endpoint.URL.String()
|
||||
}
|
||||
|
||||
// Type - returns type of endpoint.
|
||||
func (endpoint Endpoint) Type() EndpointType {
|
||||
if endpoint.Host == "" {
|
||||
return PathEndpointType
|
||||
}
|
||||
|
||||
return URLEndpointType
|
||||
}
|
||||
|
||||
// SetHTTPS - sets secure http for URLEndpointType.
|
||||
func (endpoint Endpoint) SetHTTPS() {
|
||||
if endpoint.Host != "" {
|
||||
endpoint.Scheme = "https"
|
||||
}
|
||||
}
|
||||
|
||||
// SetHTTP - sets insecure http for URLEndpointType.
|
||||
func (endpoint Endpoint) SetHTTP() {
|
||||
if endpoint.Host != "" {
|
||||
endpoint.Scheme = "http"
|
||||
}
|
||||
}
|
||||
|
||||
// NewEndpoint - returns new endpoint based on given arguments.
|
||||
func NewEndpoint(arg string) (Endpoint, error) {
|
||||
// isEmptyPath - check whether given path is not empty.
|
||||
isEmptyPath := func(path string) bool {
|
||||
return path == "" || path == "/" || path == `\`
|
||||
}
|
||||
|
||||
if isEmptyPath(arg) {
|
||||
return Endpoint{}, fmt.Errorf("empty or root endpoint is not supported")
|
||||
}
|
||||
|
||||
var isLocal bool
|
||||
u, err := url.Parse(arg)
|
||||
if err == nil && u.Host != "" {
|
||||
// URL style of endpoint.
|
||||
// Valid URL style endpoint is
|
||||
// - Scheme field must contain "http" or "https"
|
||||
// - All field should be empty except Host and Path.
|
||||
if !((u.Scheme == "http" || u.Scheme == "https") &&
|
||||
u.User == nil && u.Opaque == "" && u.ForceQuery == false && u.RawQuery == "" && u.Fragment == "") {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format")
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(u.Host)
|
||||
if err != nil {
|
||||
if !strings.Contains(err.Error(), "missing port in address") {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: %s", err)
|
||||
}
|
||||
|
||||
host = u.Host
|
||||
} else {
|
||||
var p int
|
||||
p, err = strconv.Atoi(port)
|
||||
if err != nil {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: invalid port number")
|
||||
} else if p < 1 || p > 65535 {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")
|
||||
}
|
||||
}
|
||||
|
||||
if host == "" {
|
||||
return Endpoint{}, fmt.Errorf("invalid URL endpoint format: empty host name")
|
||||
}
|
||||
|
||||
// As this is path in the URL, we should use path package, not filepath package.
|
||||
// On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo`
|
||||
u.Path = path.Clean(u.Path)
|
||||
if isEmptyPath(u.Path) {
|
||||
return Endpoint{}, fmt.Errorf("empty or root path is not supported in URL endpoint")
|
||||
}
|
||||
|
||||
isLocal, err = isLocalHost(host)
|
||||
if err != nil {
|
||||
return Endpoint{}, err
|
||||
}
|
||||
} else {
|
||||
u = &url.URL{Path: path.Clean(arg)}
|
||||
isLocal = true
|
||||
}
|
||||
|
||||
return Endpoint{
|
||||
URL: u,
|
||||
IsLocal: isLocal,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// EndpointList - list of same type of endpoint.
|
||||
type EndpointList []Endpoint
|
||||
|
||||
// Swap - helper method for sorting.
|
||||
func (endpoints EndpointList) Swap(i, j int) {
|
||||
endpoints[i], endpoints[j] = endpoints[j], endpoints[i]
|
||||
}
|
||||
|
||||
// Len - helper method for sorting.
|
||||
func (endpoints EndpointList) Len() int {
|
||||
return len(endpoints)
|
||||
}
|
||||
|
||||
// Less - helper method for sorting.
|
||||
func (endpoints EndpointList) Less(i, j int) bool {
|
||||
return endpoints[i].String() < endpoints[j].String()
|
||||
}
|
||||
|
||||
// SetHTTPS - sets secure http for URLEndpointType.
|
||||
func (endpoints EndpointList) SetHTTPS() {
|
||||
for i := range endpoints {
|
||||
endpoints[i].SetHTTPS()
|
||||
}
|
||||
}
|
||||
|
||||
// SetHTTP - sets insecure http for URLEndpointType.
|
||||
func (endpoints EndpointList) SetHTTP() {
|
||||
for i := range endpoints {
|
||||
endpoints[i].SetHTTP()
|
||||
}
|
||||
}
|
||||
|
||||
// NewEndpointList - returns new endpoint list based on input args.
|
||||
func NewEndpointList(args ...string) (endpoints EndpointList, err error) {
|
||||
// isValidDistribution - checks whether given count is a valid distribution for erasure coding.
|
||||
isValidDistribution := func(count int) bool {
|
||||
return (count >= minErasureBlocks && count <= maxErasureBlocks && count%2 == 0)
|
||||
}
|
||||
|
||||
// Check whether no. of args are valid for XL distribution.
|
||||
if !isValidDistribution(len(args)) {
|
||||
return nil, fmt.Errorf("A total of %d endpoints were found. For erasure mode it should be an even number between %d and %d", len(args), minErasureBlocks, maxErasureBlocks)
|
||||
}
|
||||
|
||||
var endpointType EndpointType
|
||||
var scheme string
|
||||
|
||||
uniqueArgs := set.NewStringSet()
|
||||
// Loop through args and adds to endpoint list.
|
||||
for i, arg := range args {
|
||||
endpoint, err := NewEndpoint(arg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("'%s': %s", arg, err.Error())
|
||||
}
|
||||
|
||||
// All endpoints have to be same type and scheme if applicable.
|
||||
if i == 0 {
|
||||
endpointType = endpoint.Type()
|
||||
scheme = endpoint.Scheme
|
||||
} else if endpoint.Type() != endpointType {
|
||||
return nil, fmt.Errorf("mixed style endpoints are not supported")
|
||||
} else if endpoint.Scheme != scheme {
|
||||
return nil, fmt.Errorf("mixed scheme is not supported")
|
||||
}
|
||||
|
||||
arg = endpoint.String()
|
||||
if uniqueArgs.Contains(arg) {
|
||||
return nil, fmt.Errorf("duplicate endpoints found")
|
||||
}
|
||||
uniqueArgs.Add(arg)
|
||||
|
||||
endpoints = append(endpoints, endpoint)
|
||||
}
|
||||
|
||||
sort.Sort(endpoints)
|
||||
|
||||
return endpoints, nil
|
||||
}
|
||||
|
||||
// CreateEndpoints - validates and creates new endpoints for given args.
|
||||
func CreateEndpoints(serverAddr string, args ...string) (string, EndpointList, SetupType, error) {
|
||||
var endpoints EndpointList
|
||||
var setupType SetupType
|
||||
var err error
|
||||
|
||||
// Check whether serverAddr is valid for this host.
|
||||
if err = CheckLocalServerAddr(serverAddr); err != nil {
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
_, serverAddrPort := mustSplitHostPort(serverAddr)
|
||||
|
||||
// For single arg, return FS setup.
|
||||
if len(args) == 1 {
|
||||
var endpoint Endpoint
|
||||
endpoint, err = NewEndpoint(args[0])
|
||||
if err != nil {
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
if endpoint.Type() != PathEndpointType {
|
||||
return serverAddr, endpoints, setupType, fmt.Errorf("use path style endpoint for FS setup")
|
||||
}
|
||||
|
||||
endpoints = append(endpoints, endpoint)
|
||||
setupType = FSSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// Convert args to endpoints
|
||||
if endpoints, err = NewEndpointList(args...); err != nil {
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
// Return XL setup when all endpoints are path style.
|
||||
if endpoints[0].Type() == PathEndpointType {
|
||||
setupType = XLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// Here all endpoints are URL style.
|
||||
endpointPathSet := set.NewStringSet()
|
||||
localEndpointCount := 0
|
||||
localServerAddrSet := set.NewStringSet()
|
||||
localPortSet := set.NewStringSet()
|
||||
for _, endpoint := range endpoints {
|
||||
endpointPathSet.Add(endpoint.Path)
|
||||
if endpoint.IsLocal {
|
||||
localServerAddrSet.Add(endpoint.Host)
|
||||
|
||||
var port string
|
||||
_, port, err = net.SplitHostPort(endpoint.Host)
|
||||
if err != nil {
|
||||
port = serverAddrPort
|
||||
}
|
||||
|
||||
localPortSet.Add(port)
|
||||
|
||||
localEndpointCount++
|
||||
}
|
||||
}
|
||||
|
||||
// No local endpoint found.
|
||||
if localEndpointCount == 0 {
|
||||
return serverAddr, endpoints, setupType, fmt.Errorf("no endpoint found for this host")
|
||||
}
|
||||
|
||||
// Check whether same path is not used in endpoints of a host.
|
||||
{
|
||||
pathIPMap := make(map[string]set.StringSet)
|
||||
for _, endpoint := range endpoints {
|
||||
var host string
|
||||
host, _, err = net.SplitHostPort(endpoint.Host)
|
||||
if err != nil {
|
||||
host = endpoint.Host
|
||||
}
|
||||
hostIPSet, _ := getHostIP4(host)
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
err = fmt.Errorf("path '%s' can not be served by different port on same address", endpoint.Path)
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
|
||||
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
|
||||
} else {
|
||||
pathIPMap[endpoint.Path] = hostIPSet
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check whether serverAddrPort matches at least in one of port used in local endpoints.
|
||||
{
|
||||
if !localPortSet.Contains(serverAddrPort) {
|
||||
if len(localPortSet) > 1 {
|
||||
err = fmt.Errorf("port number in server address must match with one of the port in local endpoints")
|
||||
} else {
|
||||
err = fmt.Errorf("server address and local endpoint have different ports")
|
||||
}
|
||||
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
}
|
||||
|
||||
// All endpoints are pointing to local host
|
||||
if len(endpoints) == localEndpointCount {
|
||||
// If all endpoints have same port number, then this is XL setup using URL style endpoints.
|
||||
if len(localPortSet) == 1 {
|
||||
if len(localServerAddrSet) > 1 {
|
||||
// TODO: Eventhough all endpoints are local, the local host is referred by different IP/name.
|
||||
// eg '172.0.0.1', 'localhost' and 'mylocalhostname' point to same local host.
|
||||
//
|
||||
// In this case, we bind to 0.0.0.0 ie to all interfaces.
|
||||
// The actual way to do is bind to only IPs in uniqueLocalHosts.
|
||||
serverAddr = net.JoinHostPort("", serverAddrPort)
|
||||
}
|
||||
|
||||
endpointPaths := endpointPathSet.ToSlice()
|
||||
endpoints, _ = NewEndpointList(endpointPaths...)
|
||||
setupType = XLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// Eventhough all endpoints are local, but those endpoints use different ports.
|
||||
// This means it is DistXL setup.
|
||||
} else {
|
||||
// This is DistXL setup.
|
||||
// Check whether local server address are not 127.x.x.x
|
||||
for _, localServerAddr := range localServerAddrSet.ToSlice() {
|
||||
host, _, err := net.SplitHostPort(localServerAddr)
|
||||
if err != nil {
|
||||
host = localServerAddr
|
||||
}
|
||||
|
||||
ipList, err := getHostIP4(host)
|
||||
fatalIf(err, "unexpected error when resolving host '%s'", host)
|
||||
|
||||
// Filter ipList by IPs those start with '127.'.
|
||||
loopBackIPs := ipList.FuncMatch(func(ip string, matchString string) bool {
|
||||
return strings.HasPrefix(ip, "127.")
|
||||
}, "")
|
||||
|
||||
// If loop back IP is found and ipList contains only loop back IPs, then error out.
|
||||
if len(loopBackIPs) > 0 && len(loopBackIPs) == len(ipList) {
|
||||
err = fmt.Errorf("'%s' resolves to loopback address is not allowed for distributed XL", localServerAddr)
|
||||
return serverAddr, endpoints, setupType, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Add missing port in all endpoints.
|
||||
for i := range endpoints {
|
||||
_, port, err := net.SplitHostPort(endpoints[i].Host)
|
||||
if err != nil {
|
||||
endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
|
||||
} else if endpoints[i].IsLocal && serverAddrPort != port {
|
||||
// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
|
||||
endpoints[i].IsLocal = false
|
||||
}
|
||||
}
|
||||
|
||||
setupType = DistXLSetupType
|
||||
return serverAddr, endpoints, setupType, nil
|
||||
}
|
||||
|
||||
// GetRemotePeers - get hosts information other than this minio service.
|
||||
func GetRemotePeers(endpoints EndpointList) []string {
|
||||
peerSet := set.NewStringSet()
|
||||
for _, endpoint := range endpoints {
|
||||
if endpoint.Type() != URLEndpointType {
|
||||
continue
|
||||
}
|
||||
|
||||
peer := endpoint.Host
|
||||
if endpoint.IsLocal {
|
||||
if _, port := mustSplitHostPort(peer); port == globalMinioPort {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
peerSet.Add(peer)
|
||||
}
|
||||
|
||||
return peerSet.ToSlice()
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewEndpoint(t *testing.T) {
|
||||
u1, _ := url.Parse("http://localhost/path")
|
||||
u2, _ := url.Parse("https://example.org/path")
|
||||
u3, _ := url.Parse("http://127.0.0.1:8080/path")
|
||||
u4, _ := url.Parse("http://192.168.253.200/path")
|
||||
|
||||
errMsg := ": no such host"
|
||||
if runtime.GOOS == "windows" {
|
||||
errMsg = ": No such host is known."
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
arg string
|
||||
expectedEndpoint Endpoint
|
||||
expectedType EndpointType
|
||||
expectedErr error
|
||||
}{
|
||||
{"foo", Endpoint{URL: &url.URL{Path: "foo"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"/foo", Endpoint{URL: &url.URL{Path: "/foo"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`\foo`, Endpoint{URL: &url.URL{Path: `\foo`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C", Endpoint{URL: &url.URL{Path: `C`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C:", Endpoint{URL: &url.URL{Path: `C:`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C:/", Endpoint{URL: &url.URL{Path: "C:"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`C:\`, Endpoint{URL: &url.URL{Path: `C:\`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`C:\foo`, Endpoint{URL: &url.URL{Path: `C:\foo`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"C:/foo", Endpoint{URL: &url.URL{Path: "C:/foo"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{`C:\\foo`, Endpoint{URL: &url.URL{Path: `C:\\foo`}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http:path", Endpoint{URL: &url.URL{Path: "http:path"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http:/path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http:///path", Endpoint{URL: &url.URL{Path: "http:/path"}, IsLocal: true}, PathEndpointType, nil},
|
||||
{"http://localhost/path", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
|
||||
{"http://localhost/path//", Endpoint{URL: u1, IsLocal: true}, URLEndpointType, nil},
|
||||
{"https://example.org/path", Endpoint{URL: u2}, URLEndpointType, nil},
|
||||
{"http://127.0.0.1:8080/path", Endpoint{URL: u3, IsLocal: true}, URLEndpointType, nil},
|
||||
{"http://192.168.253.200/path", Endpoint{URL: u4}, URLEndpointType, nil},
|
||||
{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
|
||||
{"/", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
|
||||
{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
|
||||
{"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
|
||||
{"ftp://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
|
||||
{"http://server/path?location", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")},
|
||||
{"http://:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
|
||||
{"http://:8080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: empty host name")},
|
||||
{"http://server:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")},
|
||||
{"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")},
|
||||
{"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
|
||||
{"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")},
|
||||
{"http://server/path", Endpoint{}, -1, fmt.Errorf("lookup server" + errMsg)},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
endpoint, err := NewEndpoint(testCase.arg)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else {
|
||||
var match bool
|
||||
if strings.HasSuffix(testCase.expectedErr.Error(), errMsg) {
|
||||
match = strings.HasSuffix(err.Error(), errMsg)
|
||||
} else {
|
||||
match = (testCase.expectedErr.Error() == err.Error())
|
||||
}
|
||||
if !match {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil && !reflect.DeepEqual(testCase.expectedEndpoint, endpoint) {
|
||||
t.Fatalf("endpoint: expected = %+v, got = %+v", testCase.expectedEndpoint, endpoint)
|
||||
}
|
||||
|
||||
if err == nil && testCase.expectedType != endpoint.Type() {
|
||||
t.Fatalf("type: expected = %+v, got = %+v", testCase.expectedType, endpoint.Type())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewEndpointList(t *testing.T) {
|
||||
testCases := []struct {
|
||||
args []string
|
||||
expectedErr error
|
||||
}{
|
||||
{[]string{"d1", "d2", "d3", "d4"}, nil},
|
||||
{[]string{"/d1", "/d2", "/d3", "/d4"}, nil},
|
||||
{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, nil},
|
||||
{[]string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}, nil},
|
||||
{[]string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d1", "http://example.org/d2"}, nil},
|
||||
{[]string{"https://localhost:9000/d1", "https://localhost:9001/d2", "https://localhost:9002/d3", "https://localhost:9003/d4"}, nil},
|
||||
// // It is valid WRT endpoint list that same path is expected with different port on same server.
|
||||
{[]string{"https://127.0.0.1:9000/d1", "https://127.0.0.1:9001/d1", "https://127.0.0.1:9002/d1", "https://127.0.0.1:9003/d1"}, nil},
|
||||
{[]string{"d1", "d2", "d3", "d1"}, fmt.Errorf("duplicate endpoints found")},
|
||||
{[]string{"d1", "d2", "d3", "./d1"}, fmt.Errorf("duplicate endpoints found")},
|
||||
{[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d1", "http://localhost/d4"}, fmt.Errorf("duplicate endpoints found")},
|
||||
{[]string{"d1", "d2", "d3", "d4", "d5"}, fmt.Errorf("A total of 5 endpoints were found. For erasure mode it should be an even number between 4 and 16")},
|
||||
{[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")},
|
||||
{[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")},
|
||||
{[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
_, err := NewEndpointList(testCase.args...)
|
||||
if testCase.expectedErr == nil {
|
||||
if err != nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
}
|
||||
} else if err == nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else if testCase.expectedErr.Error() != err.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateEndpoints(t *testing.T) {
|
||||
// Filter ipList by IPs those do not start with '127.'.
|
||||
nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.")
|
||||
}, "")
|
||||
if len(nonLoopBackIPs) == 0 {
|
||||
t.Fatalf("No non-loop back IP address found for this host")
|
||||
}
|
||||
nonLoopBackIP := nonLoopBackIPs.ToSlice()[0]
|
||||
|
||||
getExpectedEndpoints := func(args []string, prefix string) ([]*url.URL, []bool) {
|
||||
var URLs []*url.URL
|
||||
var localFlags []bool
|
||||
sort.Strings(args)
|
||||
for _, arg := range args {
|
||||
u, _ := url.Parse(arg)
|
||||
URLs = append(URLs, u)
|
||||
localFlags = append(localFlags, strings.HasPrefix(arg, prefix))
|
||||
}
|
||||
|
||||
return URLs, localFlags
|
||||
}
|
||||
|
||||
case1Endpoint1 := "http://" + nonLoopBackIP + "/d1"
|
||||
case1Endpoint2 := "http://" + nonLoopBackIP + "/d2"
|
||||
args := []string{
|
||||
"http://" + nonLoopBackIP + ":10000/d1",
|
||||
"http://" + nonLoopBackIP + ":10000/d2",
|
||||
"http://example.com:10000/d4",
|
||||
"http://example.org:10000/d3",
|
||||
}
|
||||
case1URLs, case1LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/")
|
||||
|
||||
case2Endpoint1 := "http://" + nonLoopBackIP + "/d1"
|
||||
case2Endpoint2 := "http://" + nonLoopBackIP + ":9000/d2"
|
||||
args = []string{
|
||||
"http://" + nonLoopBackIP + ":10000/d1",
|
||||
"http://" + nonLoopBackIP + ":9000/d2",
|
||||
"http://example.com:10000/d4",
|
||||
"http://example.org:10000/d3",
|
||||
}
|
||||
case2URLs, case2LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/")
|
||||
|
||||
case3Endpoint1 := "http://" + nonLoopBackIP + "/d1"
|
||||
args = []string{
|
||||
"http://" + nonLoopBackIP + ":80/d1",
|
||||
"http://example.com:80/d3",
|
||||
"http://example.net:80/d4",
|
||||
"http://example.org:9000/d2",
|
||||
}
|
||||
case3URLs, case3LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":80/")
|
||||
|
||||
case4Endpoint1 := "http://" + nonLoopBackIP + "/d1"
|
||||
args = []string{
|
||||
"http://" + nonLoopBackIP + ":9000/d1",
|
||||
"http://example.com:9000/d3",
|
||||
"http://example.net:9000/d4",
|
||||
"http://example.org:9000/d2",
|
||||
}
|
||||
case4URLs, case4LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/")
|
||||
|
||||
case5Endpoint1 := "http://" + nonLoopBackIP + ":9000/d1"
|
||||
case5Endpoint2 := "http://" + nonLoopBackIP + ":9001/d2"
|
||||
case5Endpoint3 := "http://" + nonLoopBackIP + ":9002/d3"
|
||||
case5Endpoint4 := "http://" + nonLoopBackIP + ":9003/d4"
|
||||
args = []string{
|
||||
case5Endpoint1,
|
||||
case5Endpoint2,
|
||||
case5Endpoint3,
|
||||
case5Endpoint4,
|
||||
}
|
||||
case5URLs, case5LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/")
|
||||
|
||||
case6Endpoint := "http://" + nonLoopBackIP + ":9003/d4"
|
||||
args = []string{
|
||||
"http://localhost:9000/d1",
|
||||
"http://localhost:9001/d2",
|
||||
"http://127.0.0.1:9002/d3",
|
||||
case6Endpoint,
|
||||
}
|
||||
case6URLs, case6LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9003/")
|
||||
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
args []string
|
||||
expectedServerAddr string
|
||||
expectedEndpoints EndpointList
|
||||
expectedSetupType SetupType
|
||||
expectedErr error
|
||||
}{
|
||||
{"localhost", []string{}, "", EndpointList{}, -1, fmt.Errorf("missing port in address localhost")},
|
||||
|
||||
// FS Setup
|
||||
{"localhost:9000", []string{"http://localhost/d1"}, "", EndpointList{}, -1, fmt.Errorf("use path style endpoint for FS setup")},
|
||||
{":443", []string{"d1"}, ":443", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{"/d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{"./d1"}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: "d1"}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{`\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `\d1`}, IsLocal: true}}, FSSetupType, nil},
|
||||
{"localhost:10000", []string{`.\d1`}, "localhost:10000", EndpointList{Endpoint{URL: &url.URL{Path: `.\d1`}, IsLocal: true}}, FSSetupType, nil},
|
||||
{":8080", []string{"https://example.org/d1", "https://example.org/d2", "https://example.org/d3", "https://example.org/d4"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
|
||||
{":8080", []string{"https://example.org/d1", "https://example.com/d2", "https://example.net:8000/d3", "https://example.edu/d1"}, "", EndpointList{}, -1, fmt.Errorf("no endpoint found for this host")},
|
||||
{"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")},
|
||||
{"localhost:9000", []string{"https://127.0.0.1:8000/d1", "https://localhost:9001/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("port number in server address must match with one of the port in local endpoints")},
|
||||
{"localhost:10000", []string{"https://127.0.0.1:8000/d1", "https://localhost:8000/d2", "https://example.com/d1", "https://example.com/d2"}, "", EndpointList{}, -1, fmt.Errorf("server address and local endpoint have different ports")},
|
||||
|
||||
// XL Setup with PathEndpointType
|
||||
{":1234", []string{"/d1", "/d2", "d3", "d4"}, ":1234",
|
||||
EndpointList{
|
||||
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "d3"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "d4"}, IsLocal: true},
|
||||
}, XLSetupType, nil},
|
||||
// XL Setup with URLEndpointType
|
||||
{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", EndpointList{
|
||||
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
|
||||
}, XLSetupType, nil},
|
||||
// XL Setup with URLEndpointType having mixed naming to local host.
|
||||
{"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, ":10000", EndpointList{
|
||||
Endpoint{URL: &url.URL{Path: "/d1"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d2"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d3"}, IsLocal: true},
|
||||
Endpoint{URL: &url.URL{Path: "/d4"}, IsLocal: true},
|
||||
}, XLSetupType, nil},
|
||||
{":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}, "", EndpointList{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")},
|
||||
|
||||
{":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d3", "http://example.com/d4"}, "", EndpointList{}, -1, fmt.Errorf("'localhost' resolves to loopback address is not allowed for distributed XL")},
|
||||
|
||||
// DistXL type
|
||||
{"127.0.0.1:10000", []string{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
|
||||
Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]},
|
||||
Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]},
|
||||
Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]},
|
||||
Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{"127.0.0.1:10000", []string{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", EndpointList{
|
||||
Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]},
|
||||
Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]},
|
||||
Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]},
|
||||
Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{":80", []string{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", EndpointList{
|
||||
Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]},
|
||||
Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]},
|
||||
Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]},
|
||||
Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{":9000", []string{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", EndpointList{
|
||||
Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]},
|
||||
Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]},
|
||||
Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]},
|
||||
Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
{":9000", []string{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}, ":9000", EndpointList{
|
||||
Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]},
|
||||
Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]},
|
||||
Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]},
|
||||
Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]},
|
||||
}, DistXLSetupType, nil},
|
||||
|
||||
// DistXL Setup using only local host.
|
||||
{":9003", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}, ":9003", EndpointList{
|
||||
Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]},
|
||||
Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]},
|
||||
Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]},
|
||||
Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]},
|
||||
}, DistXLSetupType, nil},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
serverAddr, endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
|
||||
|
||||
if err == nil {
|
||||
if testCase.expectedErr != nil {
|
||||
t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
|
||||
} else {
|
||||
if serverAddr != testCase.expectedServerAddr {
|
||||
t.Fatalf("serverAddr: expected = %v, got = %v", testCase.expectedServerAddr, serverAddr)
|
||||
}
|
||||
if !reflect.DeepEqual(endpoints, testCase.expectedEndpoints) {
|
||||
t.Fatalf("endpoints: expected = %v, got = %v", testCase.expectedEndpoints, endpoints)
|
||||
}
|
||||
if setupType != testCase.expectedSetupType {
|
||||
t.Fatalf("setupType: expected = %v, got = %v", testCase.expectedSetupType, setupType)
|
||||
}
|
||||
}
|
||||
} else if testCase.expectedErr == nil {
|
||||
t.Fatalf("error: expected = <nil>, got = %v", err)
|
||||
} else if err.Error() != testCase.expectedErr.Error() {
|
||||
t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetRemotePeers(t *testing.T) {
|
||||
tempGlobalMinioPort := globalMinioPort
|
||||
defer func() {
|
||||
globalMinioPort = tempGlobalMinioPort
|
||||
}()
|
||||
globalMinioPort = "9000"
|
||||
|
||||
testCases := []struct {
|
||||
endpointArgs []string
|
||||
expectedResult []string
|
||||
}{
|
||||
{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
|
||||
{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
|
||||
{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
endpoints, _ := NewEndpointList(testCase.endpointArgs...)
|
||||
remotePeers := GetRemotePeers(endpoints)
|
||||
if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
|
||||
t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,9 @@ import (
|
||||
// erasureCreateFile - writes an entire stream by erasure coding to
|
||||
// all the disks, writes also calculate individual block's checksum
|
||||
// for future bit-rot protection.
|
||||
func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64, dataBlocks int, parityBlocks int, algo string, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
|
||||
func erasureCreateFile(disks []StorageAPI, volume, path string, reader io.Reader, allowEmpty bool, blockSize int64,
|
||||
dataBlocks, parityBlocks int, algo HashAlgo, writeQuorum int) (bytesWritten int64, checkSums []string, err error) {
|
||||
|
||||
// Allocated blockSized buffer for reading from incoming stream.
|
||||
buf := make([]byte, blockSize)
|
||||
|
||||
|
||||
@@ -19,7 +19,9 @@ package cmd
|
||||
import "encoding/hex"
|
||||
|
||||
// Heals the erasure coded file. reedsolomon.Reconstruct() is used to reconstruct the missing parts.
|
||||
func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string, size int64, blockSize int64, dataBlocks int, parityBlocks int, algo string) (checkSums []string, err error) {
|
||||
func erasureHealFile(latestDisks []StorageAPI, outDatedDisks []StorageAPI, volume, path, healBucket, healPath string,
|
||||
size, blockSize int64, dataBlocks, parityBlocks int, algo HashAlgo) (checkSums []string, err error) {
|
||||
|
||||
var offset int64
|
||||
remainingSize := size
|
||||
|
||||
|
||||
+49
-55
@@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"io"
|
||||
"sync"
|
||||
@@ -111,7 +110,9 @@ func getReadDisks(orderedDisks []StorageAPI, index int, dataBlocks int) (readDis
|
||||
}
|
||||
|
||||
// parallelRead - reads chunks in parallel from the disks specified in []readDisks.
|
||||
func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []StorageAPI, enBlocks [][]byte, blockOffset int64, curChunkSize int64, bitRotVerify func(diskIndex int) bool, pool *bpool.BytePool) {
|
||||
func parallelRead(volume, path string, readDisks, orderedDisks []StorageAPI, enBlocks [][]byte,
|
||||
blockOffset, curChunkSize int64, brVerifiers []bitRotVerifier, pool *bpool.BytePool) {
|
||||
|
||||
// WaitGroup to synchronise the read go-routines.
|
||||
wg := &sync.WaitGroup{}
|
||||
|
||||
@@ -125,11 +126,15 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
|
||||
go func(index int) {
|
||||
defer wg.Done()
|
||||
|
||||
// Verify bit rot for the file on this disk.
|
||||
if !bitRotVerify(index) {
|
||||
// So that we don't read from this disk for the next block.
|
||||
orderedDisks[index] = nil
|
||||
return
|
||||
// evaluate if we need to perform bit-rot checking
|
||||
needBitRotVerification := true
|
||||
if brVerifiers[index].isVerified {
|
||||
needBitRotVerification = false
|
||||
// if file has bit-rot, do not reuse disk
|
||||
if brVerifiers[index].hasBitRot {
|
||||
orderedDisks[index] = nil
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
buf, err := pool.Get()
|
||||
@@ -140,7 +145,25 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
|
||||
}
|
||||
buf = buf[:curChunkSize]
|
||||
|
||||
_, err = readDisks[index].ReadFile(volume, path, blockOffset, buf)
|
||||
if needBitRotVerification {
|
||||
_, err = readDisks[index].ReadFileWithVerify(
|
||||
volume, path, blockOffset, buf,
|
||||
brVerifiers[index].algo,
|
||||
brVerifiers[index].checkSum)
|
||||
} else {
|
||||
_, err = readDisks[index].ReadFile(volume, path,
|
||||
blockOffset, buf)
|
||||
}
|
||||
|
||||
// if bit-rot verification was done, store the
|
||||
// result of verification so we can skip
|
||||
// re-doing it next time
|
||||
if needBitRotVerification {
|
||||
brVerifiers[index].isVerified = true
|
||||
_, ok := err.(hashMismatchError)
|
||||
brVerifiers[index].hasBitRot = ok
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
orderedDisks[index] = nil
|
||||
return
|
||||
@@ -153,12 +176,16 @@ func parallelRead(volume, path string, readDisks []StorageAPI, orderedDisks []St
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// erasureReadFile - read bytes from erasure coded files and writes to given writer.
|
||||
// Erasure coded files are read block by block as per given erasureInfo and data chunks
|
||||
// are decoded into a data block. Data block is trimmed for given offset and length,
|
||||
// then written to given writer. This function also supports bit-rot detection by
|
||||
// verifying checksum of individual block's checksum.
|
||||
func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path string, offset int64, length int64, totalLength int64, blockSize int64, dataBlocks int, parityBlocks int, checkSums []string, algo string, pool *bpool.BytePool) (int64, error) {
|
||||
// erasureReadFile - read bytes from erasure coded files and writes to
|
||||
// given writer. Erasure coded files are read block by block as per
|
||||
// given erasureInfo and data chunks are decoded into a data
|
||||
// block. Data block is trimmed for given offset and length, then
|
||||
// written to given writer. This function also supports bit-rot
|
||||
// detection by verifying checksum of individual block's checksum.
|
||||
func erasureReadFile(writer io.Writer, disks []StorageAPI, volume, path string,
|
||||
offset, length, totalLength, blockSize int64, dataBlocks, parityBlocks int,
|
||||
checkSums []string, algo HashAlgo, pool *bpool.BytePool) (int64, error) {
|
||||
|
||||
// Offset and length cannot be negative.
|
||||
if offset < 0 || length < 0 {
|
||||
return 0, traceError(errUnexpected)
|
||||
@@ -169,27 +196,15 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
|
||||
return 0, traceError(errUnexpected)
|
||||
}
|
||||
|
||||
// chunkSize is the amount of data that needs to be read from each disk at a time.
|
||||
// chunkSize is the amount of data that needs to be read from
|
||||
// each disk at a time.
|
||||
chunkSize := getChunkSize(blockSize, dataBlocks)
|
||||
|
||||
// bitRotVerify verifies if the file on a particular disk doesn't have bitrot
|
||||
// by verifying the hash of the contents of the file.
|
||||
bitRotVerify := func() func(diskIndex int) bool {
|
||||
verified := make([]bool, len(disks))
|
||||
// Return closure so that we have reference to []verified and
|
||||
// not recalculate the hash on it every time the function is
|
||||
// called for the same disk.
|
||||
return func(diskIndex int) bool {
|
||||
if verified[diskIndex] {
|
||||
// Already validated.
|
||||
return true
|
||||
}
|
||||
// Is this a valid block?
|
||||
isValid := isValidBlock(disks[diskIndex], volume, path, checkSums[diskIndex], algo)
|
||||
verified[diskIndex] = isValid
|
||||
return isValid
|
||||
}
|
||||
}()
|
||||
brVerifiers := make([]bitRotVerifier, len(disks))
|
||||
for i := range brVerifiers {
|
||||
brVerifiers[i].algo = algo
|
||||
brVerifiers[i].checkSum = checkSums[i]
|
||||
}
|
||||
|
||||
// Total bytes written to writer
|
||||
var bytesWritten int64
|
||||
@@ -241,7 +256,7 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
|
||||
return bytesWritten, err
|
||||
}
|
||||
// Issue a parallel read across the disks specified in readDisks.
|
||||
parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, bitRotVerify, pool)
|
||||
parallelRead(volume, path, readDisks, disks, enBlocks, blockOffset, curChunkSize, brVerifiers, pool)
|
||||
if isSuccessDecodeBlocks(enBlocks, dataBlocks) {
|
||||
// If enough blocks are available to do rs.Reconstruct()
|
||||
break
|
||||
@@ -299,27 +314,6 @@ func erasureReadFile(writer io.Writer, disks []StorageAPI, volume string, path s
|
||||
return bytesWritten, nil
|
||||
}
|
||||
|
||||
// isValidBlock - calculates the checksum hash for the block and
|
||||
// validates if its correct returns true for valid cases, false otherwise.
|
||||
func isValidBlock(disk StorageAPI, volume, path, checkSum, checkSumAlgo string) (ok bool) {
|
||||
// Disk is not available, not a valid block.
|
||||
if disk == nil {
|
||||
return false
|
||||
}
|
||||
// Checksum not available, not a valid block.
|
||||
if checkSum == "" {
|
||||
return false
|
||||
}
|
||||
// Read everything for a given block and calculate hash.
|
||||
hashWriter := newHash(checkSumAlgo)
|
||||
hashBytes, err := hashSum(disk, volume, path, hashWriter)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to calculate checksum %s/%s", volume, path)
|
||||
return false
|
||||
}
|
||||
return hex.EncodeToString(hashBytes) == checkSum
|
||||
}
|
||||
|
||||
// decodeData - decode encoded blocks.
|
||||
func decodeData(enBlocks [][]byte, dataBlocks, parityBlocks int) error {
|
||||
// Initialized reedsolomon.
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -20,7 +20,6 @@ import (
|
||||
"bytes"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"reflect"
|
||||
|
||||
@@ -195,11 +194,7 @@ func TestErasureReadUtils(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
objLayer, _, err := initObjectLayer(endpoints)
|
||||
objLayer, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
removeRoots(disks)
|
||||
t.Fatal(err)
|
||||
@@ -218,6 +213,12 @@ func (r ReadDiskDown) ReadFile(volume string, path string, offset int64, buf []b
|
||||
return 0, errFaultyDisk
|
||||
}
|
||||
|
||||
func (r ReadDiskDown) ReadFileWithVerify(volume string, path string, offset int64, buf []byte,
|
||||
algo HashAlgo, expectedHash string) (n int64, err error) {
|
||||
|
||||
return 0, errFaultyDisk
|
||||
}
|
||||
|
||||
func TestErasureReadFileDiskFail(t *testing.T) {
|
||||
// Initialize environment needed for the test.
|
||||
dataBlocks := 7
|
||||
@@ -412,7 +413,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
|
||||
}
|
||||
|
||||
// To generate random offset/length.
|
||||
r := rand.New(rand.NewSource(time.Now().UTC().UnixNano()))
|
||||
r := rand.New(rand.NewSource(UTCNow().UnixNano()))
|
||||
|
||||
// create pool buffer which will be used by erasureReadFile for
|
||||
// reading from disks and erasure decoding.
|
||||
|
||||
+18
-5
@@ -29,7 +29,7 @@ import (
|
||||
)
|
||||
|
||||
// newHashWriters - inititialize a slice of hashes for the disk count.
|
||||
func newHashWriters(diskCount int, algo string) []hash.Hash {
|
||||
func newHashWriters(diskCount int, algo HashAlgo) []hash.Hash {
|
||||
hashWriters := make([]hash.Hash, diskCount)
|
||||
for index := range hashWriters {
|
||||
hashWriters[index] = newHash(algo)
|
||||
@@ -38,13 +38,13 @@ func newHashWriters(diskCount int, algo string) []hash.Hash {
|
||||
}
|
||||
|
||||
// newHash - gives you a newly allocated hash depending on the input algorithm.
|
||||
func newHash(algo string) (h hash.Hash) {
|
||||
func newHash(algo HashAlgo) (h hash.Hash) {
|
||||
switch algo {
|
||||
case sha256Algo:
|
||||
case HashSha256:
|
||||
// sha256 checksum specially on ARM64 platforms or whenever
|
||||
// requested as dictated by `xl.json` entry.
|
||||
h = sha256.New()
|
||||
case blake2bAlgo:
|
||||
case HashBlake2b:
|
||||
// ignore the error, because New512 without a key never fails
|
||||
// New512 only returns a non-nil error, if the length of the passed
|
||||
// key > 64 bytes - but we use blake2b as hash function (no key)
|
||||
@@ -71,7 +71,7 @@ var hashBufferPool = sync.Pool{
|
||||
|
||||
// hashSum calculates the hash of the entire path and returns.
|
||||
func hashSum(disk StorageAPI, volume, path string, writer hash.Hash) ([]byte, error) {
|
||||
// Fetch staging a new staging buffer from the pool.
|
||||
// Fetch a new staging buffer from the pool.
|
||||
bufp := hashBufferPool.Get().(*[]byte)
|
||||
defer hashBufferPool.Put(bufp)
|
||||
|
||||
@@ -207,3 +207,16 @@ func copyBuffer(writer io.Writer, disk StorageAPI, volume string, path string, b
|
||||
// Success.
|
||||
return nil
|
||||
}
|
||||
|
||||
// bitRotVerifier - type representing bit-rot verification process for
|
||||
// a single under-lying object (currently whole files)
|
||||
type bitRotVerifier struct {
|
||||
// has the bit-rot verification been done?
|
||||
isVerified bool
|
||||
// is the data free of bit-rot?
|
||||
hasBitRot bool
|
||||
// hashing algorithm
|
||||
algo HashAlgo
|
||||
// hex-encoded expected raw-hash value
|
||||
checkSum string
|
||||
}
|
||||
|
||||
+58
-18
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -25,11 +25,14 @@ import (
|
||||
"net/url"
|
||||
"path"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
)
|
||||
|
||||
const (
|
||||
minioEventSource = "minio:s3"
|
||||
)
|
||||
|
||||
type externalNotifier struct {
|
||||
// Per-bucket notification config. This is updated via
|
||||
// PutBucketNotification API.
|
||||
@@ -83,11 +86,29 @@ type eventData struct {
|
||||
Bucket string
|
||||
ObjInfo ObjectInfo
|
||||
ReqParams map[string]string
|
||||
Host string
|
||||
Port string
|
||||
UserAgent string
|
||||
}
|
||||
|
||||
// New notification event constructs a new notification event message from
|
||||
// input request metadata which completed successfully.
|
||||
func newNotificationEvent(event eventData) NotificationEvent {
|
||||
getResponseOriginEndpointKey := func() string {
|
||||
host := globalMinioHost
|
||||
if host == "" {
|
||||
// FIXME: Send FQDN or hostname of this machine than sending IP address.
|
||||
host = localIP4.ToSlice()[0]
|
||||
}
|
||||
|
||||
scheme := httpScheme
|
||||
if globalIsSSL {
|
||||
scheme = httpsScheme
|
||||
}
|
||||
|
||||
return fmt.Sprintf("%s://%s:%s", scheme, host, globalMinioPort)
|
||||
}
|
||||
|
||||
// Fetch the region.
|
||||
region := serverConfig.GetRegion()
|
||||
|
||||
@@ -95,15 +116,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
creds := serverConfig.GetCredential()
|
||||
|
||||
// Time when Minio finished processing the request.
|
||||
eventTime := time.Now().UTC()
|
||||
|
||||
// API endpoint is captured here to be returned back
|
||||
// to the client for it to differentiate from which
|
||||
// server the request came from.
|
||||
var apiEndpoint string
|
||||
if len(globalAPIEndpoints) >= 1 {
|
||||
apiEndpoint = globalAPIEndpoints[0]
|
||||
}
|
||||
eventTime := UTCNow()
|
||||
|
||||
// Fetch a hexadecimal representation of event time in nano seconds.
|
||||
uniqueID := mustGetRequestID(eventTime)
|
||||
@@ -115,7 +128,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html
|
||||
nEvent := NotificationEvent{
|
||||
EventVersion: eventVersion,
|
||||
EventSource: eventSource,
|
||||
EventSource: minioEventSource,
|
||||
AwsRegion: region,
|
||||
EventTime: eventTime.Format(timeFormatAMZ),
|
||||
EventName: event.Type.String(),
|
||||
@@ -125,7 +138,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
responseRequestIDKey: uniqueID,
|
||||
// Following is a custom response element to indicate
|
||||
// event origin server endpoint.
|
||||
responseOriginEndpointKey: apiEndpoint,
|
||||
responseOriginEndpointKey: getResponseOriginEndpointKey(),
|
||||
},
|
||||
S3: eventMeta{
|
||||
SchemaVersion: eventSchemaVersion,
|
||||
@@ -136,6 +149,11 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
ARN: bucketARNPrefix + event.Bucket,
|
||||
},
|
||||
},
|
||||
Source: sourceInfo{
|
||||
Host: event.Host,
|
||||
Port: event.Port,
|
||||
UserAgent: event.UserAgent,
|
||||
},
|
||||
}
|
||||
|
||||
// Escape the object name. For example "red flower.jpg" becomes "red+flower.jpg".
|
||||
@@ -145,6 +163,7 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
if event.Type == ObjectRemovedDelete {
|
||||
nEvent.S3.Object = objectMeta{
|
||||
Key: escapedObj,
|
||||
VersionID: "1",
|
||||
Sequencer: uniqueID,
|
||||
}
|
||||
return nEvent
|
||||
@@ -152,10 +171,13 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
|
||||
// For all other events we should set ETag and Size.
|
||||
nEvent.S3.Object = objectMeta{
|
||||
Key: escapedObj,
|
||||
ETag: event.ObjInfo.MD5Sum,
|
||||
Size: event.ObjInfo.Size,
|
||||
Sequencer: uniqueID,
|
||||
Key: escapedObj,
|
||||
ETag: event.ObjInfo.ETag,
|
||||
Size: event.ObjInfo.Size,
|
||||
ContentType: event.ObjInfo.ContentType,
|
||||
UserDefined: event.ObjInfo.UserDefined,
|
||||
VersionID: "1",
|
||||
Sequencer: uniqueID,
|
||||
}
|
||||
|
||||
// Success.
|
||||
@@ -630,7 +652,6 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
if !webhookN.Enable {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := addQueueTarget(queueTargets, accountID, queueTypeWebhook, newWebhookNotify); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -674,6 +695,25 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// Load MySQL targets, initialize their respective loggers.
|
||||
for accountID, msqlN := range serverConfig.Notify.GetMySQL() {
|
||||
if !msqlN.Enable {
|
||||
continue
|
||||
}
|
||||
|
||||
if queueARN, err := addQueueTarget(queueTargets, accountID, queueTypeMySQL, newMySQLNotify); err != nil {
|
||||
if _, ok := err.(net.Error); ok {
|
||||
err = &net.OpError{
|
||||
Op: "Connecting to " + queueARN,
|
||||
Net: "tcp",
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Load Kafka targets, initialize their respective loggers.
|
||||
for accountID, kafkaN := range serverConfig.Notify.GetKafka() {
|
||||
if !kafkaN.Enable {
|
||||
|
||||
+14
-48
@@ -19,7 +19,6 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"net"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -40,17 +39,13 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
defer removeRoots(disks)
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
bucketName := "bucket"
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected error:", err)
|
||||
}
|
||||
|
||||
@@ -97,11 +92,7 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -128,11 +119,7 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -159,11 +146,7 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -190,11 +173,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -221,11 +200,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -252,11 +227,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unable to create directories for FS backend. ", err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(disks)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs, _, err := initObjectLayer(endpoints)
|
||||
fs, _, err := initObjectLayer(mustGetNewEndpointList(disks...))
|
||||
if err != nil {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
@@ -276,15 +247,10 @@ func (s *TestPeerRPCServerData) Setup(t *testing.T) {
|
||||
s.testServer = StartTestPeersRPCServer(t, s.serverType)
|
||||
|
||||
// setup port and minio addr
|
||||
host, port, err := net.SplitHostPort(s.testServer.Server.Listener.Addr().String())
|
||||
if err != nil {
|
||||
t.Fatalf("Initialisation error: %v", err)
|
||||
}
|
||||
host, port := mustSplitHostPort(s.testServer.Server.Listener.Addr().String())
|
||||
globalMinioHost = host
|
||||
globalMinioPort = port
|
||||
globalMinioAddr = getLocalAddress(
|
||||
s.testServer.SrvCmdCfg,
|
||||
)
|
||||
globalMinioAddr = getEndpointsLocalAddr(s.testServer.endpoints)
|
||||
|
||||
// initialize the peer client(s)
|
||||
initGlobalS3Peers(s.testServer.Disks)
|
||||
@@ -377,7 +343,7 @@ func TestInitEventNotifier(t *testing.T) {
|
||||
}
|
||||
|
||||
// create bucket
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected error:", err)
|
||||
}
|
||||
|
||||
@@ -442,7 +408,7 @@ func TestListenBucketNotification(t *testing.T) {
|
||||
objectName := "object"
|
||||
|
||||
// Create the bucket to listen on
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected error:", err)
|
||||
}
|
||||
|
||||
@@ -552,12 +518,12 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
|
||||
|
||||
// Make a bucket to store topicConfigs.
|
||||
randBucket := getRandomBucketName()
|
||||
if err := obj.MakeBucket(randBucket); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(randBucket, ""); err != nil {
|
||||
t.Fatalf("Failed to make bucket %s", randBucket)
|
||||
}
|
||||
|
||||
// Add a topicConfig to an empty notificationConfig.
|
||||
accountID := fmt.Sprintf("%d", time.Now().UTC().UnixNano())
|
||||
accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
|
||||
accountARN := fmt.Sprintf(
|
||||
"arn:minio:sqs:%s:%s:listen-%s",
|
||||
serverConfig.GetRegion(),
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
)
|
||||
|
||||
// FileLogger - file logger which logs to a file.
|
||||
type FileLogger struct {
|
||||
BaseLogTarget
|
||||
Filename string `json:"filename"`
|
||||
file *os.File
|
||||
}
|
||||
|
||||
// Fire - log entry handler.
|
||||
func (logger FileLogger) Fire(entry *logrus.Entry) (err error) {
|
||||
if !logger.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
msgBytes, err := logger.formatter.Format(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = logger.file.Write(msgBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = logger.file.Sync()
|
||||
return err
|
||||
}
|
||||
|
||||
// String - represents ConsoleLogger as string.
|
||||
func (logger FileLogger) String() string {
|
||||
enableStr := "disabled"
|
||||
if logger.Enable {
|
||||
enableStr = "enabled"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("file:%s:%s", enableStr, logger.Filename)
|
||||
}
|
||||
|
||||
// NewFileLogger - creates new file logger object.
|
||||
func NewFileLogger(filename string) (logger FileLogger) {
|
||||
logger.Enable = true
|
||||
logger.formatter = new(logrus.JSONFormatter)
|
||||
logger.Filename = filename
|
||||
|
||||
return logger
|
||||
}
|
||||
|
||||
// InitFileLogger - initializes file logger.
|
||||
func InitFileLogger(logger *FileLogger) (err error) {
|
||||
if !logger.Enable {
|
||||
return err
|
||||
}
|
||||
|
||||
if logger.formatter == nil {
|
||||
logger.formatter = new(logrus.JSONFormatter)
|
||||
}
|
||||
|
||||
if logger.file == nil {
|
||||
logger.file, err = os.OpenFile(logger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0664)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
+166
-22
@@ -20,8 +20,12 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// fsFormat - structure holding 'fs' format.
|
||||
@@ -29,6 +33,15 @@ type fsFormat struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// FS format version strings.
|
||||
const (
|
||||
// Represents the current backend disk structure
|
||||
// version under `.minio.sys` and actual data namespace.
|
||||
|
||||
// formatConfigV1.fsFormat.Version
|
||||
fsFormatBackendV1 = "1"
|
||||
)
|
||||
|
||||
// xlFormat - structure holding 'xl' format.
|
||||
type xlFormat struct {
|
||||
Version string `json:"version"` // Version of 'xl' format.
|
||||
@@ -38,6 +51,15 @@ type xlFormat struct {
|
||||
JBOD []string `json:"jbod"`
|
||||
}
|
||||
|
||||
// XL format version strings.
|
||||
const (
|
||||
// Represents the current backend disk structure
|
||||
// version under `.minio.sys` and actual data namespace.
|
||||
|
||||
// formatConfigV1.xlFormat.Version
|
||||
xlFormatBackendV1 = "1"
|
||||
)
|
||||
|
||||
// formatConfigV1 - structure holds format config version '1'.
|
||||
type formatConfigV1 struct {
|
||||
Version string `json:"version"` // Version of the format config.
|
||||
@@ -47,6 +69,120 @@ type formatConfigV1 struct {
|
||||
XL *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
|
||||
}
|
||||
|
||||
// Format json file.
|
||||
const (
|
||||
// Format config file carries backend format specific details.
|
||||
formatConfigFile = "format.json"
|
||||
|
||||
// Format config tmp file carries backend format.
|
||||
formatConfigFileTmp = "format.json.tmp"
|
||||
)
|
||||
|
||||
// `format.json` version value.
|
||||
const (
|
||||
// formatConfigV1.Version represents the version string
|
||||
// of the current structure and its fields in `format.json`.
|
||||
formatFileV1 = "1"
|
||||
|
||||
// Future `format.json` structure changes should have
|
||||
// its own version and should be subsequently listed here.
|
||||
)
|
||||
|
||||
// Constitutes `format.json` backend name.
|
||||
const (
|
||||
// Represents FS backend.
|
||||
formatBackendFS = "fs"
|
||||
|
||||
// Represents XL backend.
|
||||
formatBackendXL = "xl"
|
||||
)
|
||||
|
||||
// CheckFS if the format is FS and is valid with right values
|
||||
// returns appropriate errors otherwise.
|
||||
func (f *formatConfigV1) CheckFS() error {
|
||||
// Validate if format config version is v1.
|
||||
if f.Version != formatFileV1 {
|
||||
return fmt.Errorf("Unknown format file version '%s'", f.Version)
|
||||
}
|
||||
|
||||
// Validate if we have the expected format.
|
||||
if f.Format != formatBackendFS {
|
||||
return fmt.Errorf("FS backend format required. Found '%s'", f.Format)
|
||||
}
|
||||
|
||||
// Check if format is currently supported.
|
||||
if f.FS.Version != fsFormatBackendV1 {
|
||||
return fmt.Errorf("Unknown backend FS format version '%s'", f.FS.Version)
|
||||
}
|
||||
|
||||
// Success.
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadFormat - loads format config v1, returns `errUnformattedDisk`
|
||||
// if reading format.json fails with io.EOF.
|
||||
func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error {
|
||||
_, err := f.ReadFrom(lk)
|
||||
if errorCause(err) == io.EOF {
|
||||
// No data on disk `format.json` still empty
|
||||
// treat it as unformatted disk.
|
||||
return traceError(errUnformattedDisk)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
|
||||
// Serialize to prepare to write to disk.
|
||||
var fbytes []byte
|
||||
fbytes, err = json.Marshal(f)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
if err = lk.Truncate(0); err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
_, err = lk.Write(fbytes)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
return int64(len(fbytes)), nil
|
||||
}
|
||||
|
||||
func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
|
||||
var fbytes []byte
|
||||
fi, err := lk.Stat()
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
if len(fbytes) == 0 {
|
||||
return 0, traceError(io.EOF)
|
||||
}
|
||||
// Decode `format.json`.
|
||||
if err = json.Unmarshal(fbytes, f); err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
return int64(len(fbytes)), nil
|
||||
}
|
||||
|
||||
func newFSFormat() (format *formatConfigV1) {
|
||||
return newFSFormatV1()
|
||||
}
|
||||
|
||||
// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
|
||||
func newFSFormatV1() (format *formatConfigV1) {
|
||||
return &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
|
||||
All disks online
|
||||
@@ -768,31 +904,39 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
|
||||
return reorderDisks(bootstrapDisks, formatConfigs)
|
||||
}
|
||||
|
||||
func checkFormatXLValues(formatConfigs []*formatConfigV1) error {
|
||||
for _, formatXL := range formatConfigs {
|
||||
if formatXL == nil {
|
||||
continue
|
||||
}
|
||||
// Validate format version and format type.
|
||||
if formatXL.Version != "1" {
|
||||
return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
|
||||
}
|
||||
if formatXL.Format != "xl" {
|
||||
return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
|
||||
}
|
||||
if formatXL.XL.Version != "1" {
|
||||
return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
|
||||
}
|
||||
if len(formatConfigs) != len(formatXL.XL.JBOD) {
|
||||
return fmt.Errorf("Number of disks %d did not match the backend format %d", len(formatConfigs), len(formatXL.XL.JBOD))
|
||||
}
|
||||
func checkFormatXLValue(formatXL *formatConfigV1) error {
|
||||
// Validate format version and format type.
|
||||
if formatXL.Version != formatFileV1 {
|
||||
return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
|
||||
}
|
||||
if formatXL.Format != formatBackendXL {
|
||||
return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
|
||||
}
|
||||
if formatXL.XL.Version != "1" {
|
||||
return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFormatXLValues(formatConfigs []*formatConfigV1) (int, error) {
|
||||
for i, formatXL := range formatConfigs {
|
||||
if formatXL == nil {
|
||||
continue
|
||||
}
|
||||
if err := checkFormatXLValue(formatXL); err != nil {
|
||||
return i, err
|
||||
}
|
||||
if len(formatConfigs) != len(formatXL.XL.JBOD) {
|
||||
return i, fmt.Errorf("Number of disks %d did not match the backend format %d",
|
||||
len(formatConfigs), len(formatXL.XL.JBOD))
|
||||
}
|
||||
}
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
// checkFormatXL - verifies if format.json format is intact.
|
||||
func checkFormatXL(formatConfigs []*formatConfigV1) error {
|
||||
if err := checkFormatXLValues(formatConfigs); err != nil {
|
||||
if _, err := checkFormatXLValues(formatConfigs); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkJBODConsistency(formatConfigs); err != nil {
|
||||
@@ -867,10 +1011,10 @@ func initFormatXL(storageDisks []StorageAPI) (err error) {
|
||||
}
|
||||
// Allocate format config.
|
||||
formats[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: mustGetUUID(),
|
||||
},
|
||||
}
|
||||
|
||||
+224
-145
@@ -18,7 +18,12 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// generates a valid format.json for XL backend.
|
||||
@@ -30,10 +35,10 @@ func genFormatXLValid() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -51,10 +56,10 @@ func genFormatXLInvalidVersion() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -75,10 +80,10 @@ func genFormatXLInvalidFormat() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -99,10 +104,10 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -116,8 +121,8 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
|
||||
|
||||
func genFormatFS() *formatConfigV1 {
|
||||
return &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "fs",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -130,10 +135,10 @@ func genFormatXLInvalidJBODCount() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -151,10 +156,10 @@ func genFormatXLInvalidJBOD() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -178,10 +183,10 @@ func genFormatXLInvalidDisks() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -202,10 +207,10 @@ func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -223,7 +228,7 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
|
||||
var err error
|
||||
xl := obj.(*xlObjects)
|
||||
|
||||
err = obj.MakeBucket("bucket")
|
||||
err = obj.MakeBucketWithLocation("bucket", "")
|
||||
if err != nil {
|
||||
return []StorageAPI{}, err
|
||||
}
|
||||
@@ -240,10 +245,10 @@ func prepareFormatXLHealFreshDisks(obj ObjectLayer) ([]StorageAPI, error) {
|
||||
// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
|
||||
// created when minio starts
|
||||
for i := 3; i <= 5; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
return []StorageAPI{}, err
|
||||
}
|
||||
if err = xl.storageDisks[i].DeleteFile(".minio.sys", "tmp"); err != nil {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, "tmp"); err != nil {
|
||||
return []StorageAPI{}, err
|
||||
}
|
||||
if err = xl.storageDisks[i].DeleteFile(bucket, object+"/xl.json"); err != nil {
|
||||
@@ -273,12 +278,8 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -309,12 +310,8 @@ func TestFormatXLHealFreshDisksErrorExpected(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
@@ -354,7 +351,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
|
||||
|
||||
xl := obj.(*xlObjects)
|
||||
|
||||
err = obj.MakeBucket("bucket")
|
||||
err = obj.MakeBucketWithLocation("bucket", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -369,19 +366,19 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
|
||||
}
|
||||
|
||||
// Now, remove two format files.. Load them and reorder
|
||||
if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[3].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = xl.storageDisks[11].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[11].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove the content of export dir 10 but preserve .minio.sys because it is automatically
|
||||
// created when minio starts
|
||||
if err = xl.storageDisks[10].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[10].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = xl.storageDisks[10].DeleteFile(".minio.sys", "tmp"); err != nil {
|
||||
if err = xl.storageDisks[10].DeleteFile(minioMetaBucket, "tmp"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = xl.storageDisks[10].DeleteFile(bucket, object+"/xl.json"); err != nil {
|
||||
@@ -427,7 +424,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
|
||||
|
||||
xl := obj.(*xlObjects)
|
||||
|
||||
err = obj.MakeBucket("bucket")
|
||||
err = obj.MakeBucketWithLocation("bucket", "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -442,10 +439,10 @@ func TestFormatXLReorderByInspection(t *testing.T) {
|
||||
}
|
||||
|
||||
// Now, remove two format files.. Load them and reorder
|
||||
if err = xl.storageDisks[3].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[3].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = xl.storageDisks[5].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[5].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
@@ -563,10 +560,10 @@ func TestSavedUUIDOrder(t *testing.T) {
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "xl",
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: "1",
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
@@ -596,12 +593,8 @@ func TestInitFormatXLErrors(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -694,6 +687,163 @@ func TestGenericFormatCheckXL(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSCheckFormatFSErr - test loadFormatFS loading older format.
|
||||
func TestFSCheckFormatFSErr(t *testing.T) {
|
||||
// Prepare for testing
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer removeAll(disk)
|
||||
|
||||
// Assign a new UUID.
|
||||
uuid := mustGetUUID()
|
||||
|
||||
// Initialize meta volume, if volume already exists ignores it.
|
||||
if err := initMetaVolumeFS(disk, uuid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
format *formatConfigV1
|
||||
formatWriteErr error
|
||||
formatCheckErr error
|
||||
shouldPass bool
|
||||
}{
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
},
|
||||
formatCheckErr: nil,
|
||||
shouldPass: true,
|
||||
},
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: "10",
|
||||
},
|
||||
},
|
||||
formatCheckErr: errors.New("Unknown backend FS format version '10'"),
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: "garbage",
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
},
|
||||
formatCheckErr: errors.New("FS backend format required. Found 'garbage'"),
|
||||
},
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: "-1",
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
},
|
||||
formatCheckErr: errors.New("Unknown format file version '-1'"),
|
||||
},
|
||||
}
|
||||
|
||||
fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
|
||||
for i, testCase := range testCases {
|
||||
lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = testCase.format.WriteTo(lk)
|
||||
lk.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Expected nil, got %s", i+1, err)
|
||||
}
|
||||
|
||||
lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
formatCfg := &formatConfigV1{}
|
||||
_, err = formatCfg.ReadFrom(lk)
|
||||
lk.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = formatCfg.CheckFS()
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Should not fail with unexpected %s, expected nil", i+1, err)
|
||||
}
|
||||
if err == nil && !testCase.shouldPass {
|
||||
t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr)
|
||||
}
|
||||
if err != nil && !testCase.shouldPass {
|
||||
if errorCause(err).Error() != testCase.formatCheckErr.Error() {
|
||||
t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSCheckFormatFS - test loadFormatFS with healty and faulty disks
|
||||
func TestFSCheckFormatFS(t *testing.T) {
|
||||
// Prepare for testing
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer removeAll(disk)
|
||||
|
||||
// Assign a new UUID.
|
||||
uuid := mustGetUUID()
|
||||
|
||||
// Initialize meta volume, if volume already exists ignores it.
|
||||
if err := initMetaVolumeFS(disk, uuid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
|
||||
lk, err := lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
format := newFSFormatV1()
|
||||
_, err = format.WriteTo(lk)
|
||||
lk.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Loading corrupted format file
|
||||
file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
t.Fatal("Should not fail here", err)
|
||||
}
|
||||
file.Write([]byte{'b'})
|
||||
file.Close()
|
||||
|
||||
lk, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
format = &formatConfigV1{}
|
||||
_, err = format.ReadFrom(lk)
|
||||
lk.Close()
|
||||
if err == nil {
|
||||
t.Fatal("Should return an error here")
|
||||
}
|
||||
|
||||
// Loading format file from disk not found.
|
||||
removeAll(disk)
|
||||
_, err = lock.LockedOpenFile(preparePath(fsFormatPath), os.O_RDONLY, 0600)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
t.Fatal("Should return 'format.json' does not exist, but got", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFormatXLErrs(t *testing.T) {
|
||||
nDisks := 16
|
||||
fsDirs, err := getRandomDisks(nDisks)
|
||||
@@ -702,12 +852,8 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Create an instance of xl backend.
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -733,11 +879,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -761,11 +903,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -773,7 +911,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
|
||||
// disks 0..10 returns unformatted disk
|
||||
for i := 0; i <= 10; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -787,11 +925,7 @@ func TestLoadFormatXLErrs(t *testing.T) {
|
||||
}
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -820,13 +954,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -842,13 +971,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -866,13 +990,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -892,13 +1011,8 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -914,19 +1028,14 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 15; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -940,19 +1049,14 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Corrupted format json in one disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 15; i++ {
|
||||
if err = xl.storageDisks[i].AppendFile(".minio.sys", "format.json", []byte("corrupted data")); err != nil {
|
||||
if err = xl.storageDisks[i].AppendFile(minioMetaBucket, formatConfigFile, []byte("corrupted data")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -976,13 +1080,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err := parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Everything is fine, should return nil
|
||||
obj, _, err := initObjectLayer(endpoints)
|
||||
obj, _, err := initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -997,13 +1096,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Disks 0..15 are nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1021,13 +1115,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk returns Faulty Disk
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1047,13 +1136,8 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// One disk is not found, heal corrupted disks should return nil
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -1069,19 +1153,14 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
endpoints, err = parseStorageEndpoints(fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Remove format.json of all disks
|
||||
obj, _, err = initObjectLayer(endpoints)
|
||||
obj, _, err = initObjectLayer(mustGetNewEndpointList(fsDirs...))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
xl = obj.(*xlObjects)
|
||||
for i := 0; i <= 15; i++ {
|
||||
if err = xl.storageDisks[i].DeleteFile(".minio.sys", "format.json"); err != nil {
|
||||
if err = xl.storageDisks[i].DeleteFile(minioMetaBucket, formatConfigFile); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
+105
-23
@@ -20,6 +20,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
pathutil "path"
|
||||
"runtime"
|
||||
)
|
||||
|
||||
// Removes only the file at given path does not remove
|
||||
@@ -122,18 +123,27 @@ func fsMkdir(dirPath string) (err error) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Lookup if directory exists, returns directory
|
||||
// attributes upon success.
|
||||
func fsStatDir(statDir string) (os.FileInfo, error) {
|
||||
if statDir == "" {
|
||||
func fsStat(statLoc string) (os.FileInfo, error) {
|
||||
if statLoc == "" {
|
||||
return nil, traceError(errInvalidArgument)
|
||||
}
|
||||
if err := checkPathLength(statDir); err != nil {
|
||||
if err := checkPathLength(statLoc); err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
fi, err := osStat(preparePath(statLoc))
|
||||
if err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
|
||||
fi, err := os.Stat(preparePath(statDir))
|
||||
return fi, nil
|
||||
}
|
||||
|
||||
// Lookup if directory exists, returns directory
|
||||
// attributes upon success.
|
||||
func fsStatDir(statDir string) (os.FileInfo, error) {
|
||||
fi, err := fsStat(statDir)
|
||||
if err != nil {
|
||||
err = errorCause(err)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, traceError(errVolumeNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
@@ -151,16 +161,9 @@ func fsStatDir(statDir string) (os.FileInfo, error) {
|
||||
|
||||
// Lookup if file exists, returns file attributes upon success
|
||||
func fsStatFile(statFile string) (os.FileInfo, error) {
|
||||
if statFile == "" {
|
||||
return nil, traceError(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err := checkPathLength(statFile); err != nil {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
|
||||
fi, err := os.Stat(preparePath(statFile))
|
||||
fi, err := fsStat(statFile)
|
||||
if err != nil {
|
||||
err = errorCause(err)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, traceError(errFileNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
@@ -173,7 +176,7 @@ func fsStatFile(statFile string) (os.FileInfo, error) {
|
||||
return nil, traceError(err)
|
||||
}
|
||||
if fi.IsDir() {
|
||||
return nil, traceError(errFileNotFound)
|
||||
return nil, traceError(errFileAccessDenied)
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
@@ -205,7 +208,7 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
|
||||
}
|
||||
|
||||
// Stat to get the size of the file at path.
|
||||
st, err := fr.Stat()
|
||||
st, err := osStat(preparePath(readPath))
|
||||
if err != nil {
|
||||
return nil, 0, traceError(err)
|
||||
}
|
||||
@@ -229,7 +232,7 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
|
||||
|
||||
// Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer.
|
||||
func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) {
|
||||
if filePath == "" || reader == nil || buf == nil {
|
||||
if filePath == "" || reader == nil {
|
||||
return 0, traceError(errInvalidArgument)
|
||||
}
|
||||
|
||||
@@ -262,11 +265,18 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
|
||||
}
|
||||
}
|
||||
|
||||
bytesWritten, err := io.CopyBuffer(writer, reader, buf)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
var bytesWritten int64
|
||||
if buf != nil {
|
||||
bytesWritten, err = io.CopyBuffer(writer, reader, buf)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
} else {
|
||||
bytesWritten, err = io.Copy(writer, reader)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
}
|
||||
|
||||
return bytesWritten, nil
|
||||
}
|
||||
|
||||
@@ -275,6 +285,12 @@ func fsRemoveUploadIDPath(basePath, uploadIDPath string) error {
|
||||
if basePath == "" || uploadIDPath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
}
|
||||
if err := checkPathLength(basePath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
if err := checkPathLength(uploadIDPath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
// List all the entries in uploadID.
|
||||
entries, err := readDir(uploadIDPath)
|
||||
@@ -318,6 +334,26 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) {
|
||||
// Renames source path to destination path, creates all the
|
||||
// missing parents if they don't exist.
|
||||
func fsRenameFile(sourcePath, destPath string) error {
|
||||
if err := checkPathLength(sourcePath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
if err := checkPathLength(destPath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
// Verify if source path exists.
|
||||
if _, err := os.Stat(preparePath(sourcePath)); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return traceError(errFileNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
return traceError(errFileAccessDenied)
|
||||
} else if isSysErrPathNotFound(err) {
|
||||
return traceError(errFileNotFound)
|
||||
} else if isSysErrNotDir(err) {
|
||||
// File path cannot be verified since one of the parents is a file.
|
||||
return traceError(errFileAccessDenied)
|
||||
}
|
||||
return traceError(err)
|
||||
}
|
||||
if err := mkdirAll(pathutil.Dir(destPath), 0777); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
@@ -343,7 +379,7 @@ func fsDeleteFile(basePath, deletePath string) error {
|
||||
}
|
||||
|
||||
// Verify if the path exists.
|
||||
pathSt, err := os.Stat(preparePath(deletePath))
|
||||
pathSt, err := osStat(preparePath(deletePath))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return traceError(errFileNotFound)
|
||||
@@ -377,3 +413,49 @@ func fsDeleteFile(basePath, deletePath string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fsRemoveMeta safely removes a locked file and takes care of Windows special case
|
||||
func fsRemoveMeta(basePath, deletePath, tmpDir string) error {
|
||||
// Special case for windows please read through.
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
// Ordinarily windows does not permit deletion or renaming of files still
|
||||
// in use, but if all open handles to that file were opened with FILE_SHARE_DELETE
|
||||
// then it can permit renames and deletions of open files.
|
||||
//
|
||||
// There are however some gotchas with this, and it is worth listing them here.
|
||||
// Firstly, Windows never allows you to really delete an open file, rather it is
|
||||
// flagged as delete pending and its entry in its directory remains visible
|
||||
// (though no new file handles may be opened to it) and when the very last
|
||||
// open handle to the file in the system is closed, only then is it truly
|
||||
// deleted. Well, actually only sort of truly deleted, because Windows only
|
||||
// appears to remove the file entry from the directory, but in fact that
|
||||
// entry is merely hidden and actually still exists and attempting to create
|
||||
// a file with the same name will return an access denied error. How long it
|
||||
// silently exists for depends on a range of factors, but put it this way:
|
||||
// if your code loops creating and deleting the same file name as you might
|
||||
// when operating a lock file, you're going to see lots of random spurious
|
||||
// access denied errors and truly dismal lock file performance compared to POSIX.
|
||||
//
|
||||
// We work-around these un-POSIX file semantics by taking a dual step to
|
||||
// deleting files. Firstly, it renames the file to tmp location into multipartTmpBucket
|
||||
// We always open files with FILE_SHARE_DELETE permission enabled, with that
|
||||
// flag Windows permits renaming and deletion, and because the name was changed
|
||||
// to a very random name somewhere not in its origin directory before deletion,
|
||||
// you don't see those unexpected random errors when creating files with the
|
||||
// same name as a recently deleted file as you do anywhere else on Windows.
|
||||
// Because the file is probably not in its original containing directory any more,
|
||||
// deletions of that directory will not fail with "directory not empty" as they
|
||||
// otherwise normally would either.
|
||||
|
||||
tmpPath := pathJoin(tmpDir, mustGetUUID())
|
||||
|
||||
fsRenameFile(deletePath, tmpPath)
|
||||
|
||||
// Proceed to deleting the directory if empty
|
||||
fsDeleteFile(basePath, pathutil.Dir(deletePath))
|
||||
|
||||
// Finally delete the renamed file.
|
||||
return fsDeleteFile(tmpDir, tmpPath)
|
||||
}
|
||||
return fsDeleteFile(basePath, deletePath)
|
||||
}
|
||||
|
||||
+87
-11
@@ -18,9 +18,39 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
func TestFSRenameFile(t *testing.T) {
|
||||
// create posix test setup
|
||||
_, path, err := newPosixTestSetup()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create posix test setup, %s", err)
|
||||
}
|
||||
defer removeAll(path)
|
||||
|
||||
if err = fsMkdir(pathJoin(path, "testvolume1")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNotFound {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSStats(t *testing.T) {
|
||||
// create posix test setup
|
||||
_, path, err := newPosixTestSetup()
|
||||
@@ -43,9 +73,8 @@ func TestFSStats(t *testing.T) {
|
||||
t.Fatalf("Unable to create volume, %s", err)
|
||||
}
|
||||
|
||||
var buf = make([]byte, 4096)
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -55,7 +84,7 @@ func TestFSStats(t *testing.T) {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, buf, reader.Size()); err != nil {
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -105,7 +134,7 @@ func TestFSStats(t *testing.T) {
|
||||
srcFSPath: path,
|
||||
srcVol: "success-vol",
|
||||
srcPath: "path",
|
||||
expectedErr: errFileNotFound,
|
||||
expectedErr: errFileAccessDenied,
|
||||
},
|
||||
// Test case - 6.
|
||||
// Test case with src path segment > 255.
|
||||
@@ -138,7 +167,8 @@ func TestFSStats(t *testing.T) {
|
||||
|
||||
for i, testCase := range testCases {
|
||||
if testCase.srcPath != "" {
|
||||
if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
|
||||
if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol,
|
||||
testCase.srcPath)); errorCause(err) != testCase.expectedErr {
|
||||
t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
} else {
|
||||
@@ -169,9 +199,8 @@ func TestFSCreateAndOpen(t *testing.T) {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
var buf = make([]byte, 4096)
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -199,7 +228,7 @@ func TestFSCreateAndOpen(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, buf, reader.Size())
|
||||
_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
|
||||
if errorCause(err) != testCase.expectedErr {
|
||||
t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
@@ -292,15 +321,14 @@ func TestFSRemoves(t *testing.T) {
|
||||
t.Fatalf("Unable to create directory, %s", err)
|
||||
}
|
||||
|
||||
var buf = make([]byte, 4096)
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
reader.Seek(0, 0)
|
||||
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, buf, reader.Size()); err != nil {
|
||||
if _, err = fsCreateFile(pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -396,3 +424,51 @@ func TestFSRemoves(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSRemoveMeta(t *testing.T) {
|
||||
// create posix test setup
|
||||
_, fsPath, err := newPosixTestSetup()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create posix test setup, %s", err)
|
||||
}
|
||||
defer removeAll(fsPath)
|
||||
|
||||
// Setup test environment.
|
||||
if err = fsMkdir(pathJoin(fsPath, "success-vol")); err != nil {
|
||||
t.Fatalf("Unable to create directory, %s", err)
|
||||
}
|
||||
|
||||
filePath := pathJoin(fsPath, "success-vol", "success-file")
|
||||
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(filePath, reader, nil, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
|
||||
rwPool := &fsIOPool{
|
||||
readersMap: make(map[string]*lock.RLockedFile),
|
||||
}
|
||||
|
||||
if _, err := rwPool.Open(filePath); err != nil {
|
||||
t.Fatalf("Unable to lock file %s", filePath)
|
||||
}
|
||||
|
||||
defer rwPool.Close(filePath)
|
||||
|
||||
tmpDir, tmpErr := ioutil.TempDir(globalTestTmpDir, "minio-")
|
||||
if tmpErr != nil {
|
||||
t.Fatal(tmpErr)
|
||||
}
|
||||
|
||||
if err := fsRemoveMeta(fsPath, filePath, tmpDir); err != nil {
|
||||
t.Fatalf("Unable to remove file, %s", err)
|
||||
}
|
||||
|
||||
if _, err := osStat(preparePath(filePath)); !os.IsNotExist(err) {
|
||||
t.Fatalf("`%s` file found though it should have been deleted.", filePath)
|
||||
}
|
||||
|
||||
if _, err := osStat(preparePath(path.Dir(filePath))); !os.IsNotExist(err) {
|
||||
t.Fatalf("`%s` parent directory found though it should have been deleted.", filePath)
|
||||
}
|
||||
}
|
||||
|
||||
+168
-61
@@ -24,14 +24,31 @@ import (
|
||||
pathutil "path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// FS format, and object metadata.
|
||||
const (
|
||||
fsMetaJSONFile = "fs.json"
|
||||
fsFormatJSONFile = "format.json"
|
||||
// fs.json object metadata.
|
||||
fsMetaJSONFile = "fs.json"
|
||||
)
|
||||
|
||||
// FS metadata constants.
|
||||
const (
|
||||
// FS backend meta 1.0.0 version.
|
||||
fsMetaVersion100 = "1.0.0"
|
||||
|
||||
// FS backend meta 1.0.1 version.
|
||||
fsMetaVersion = "1.0.1"
|
||||
|
||||
// FS backend meta format.
|
||||
fsMetaFormat = "fs"
|
||||
|
||||
// Add more constants here.
|
||||
)
|
||||
|
||||
// A fsMetaV1 represents a metadata header mapping keys to sets of values.
|
||||
@@ -46,6 +63,19 @@ type fsMetaV1 struct {
|
||||
Parts []objectPartInfo `json:"parts,omitempty"`
|
||||
}
|
||||
|
||||
// IsValid - tells if the format is sane by validating the version
|
||||
// string and format style.
|
||||
func (m fsMetaV1) IsValid() bool {
|
||||
return isFSMetaValid(m.Version, m.Format)
|
||||
}
|
||||
|
||||
// Verifies if the backend format metadata is sane by validating
|
||||
// the version string and format style.
|
||||
func isFSMetaValid(version, format string) bool {
|
||||
return ((version == fsMetaVersion || version == fsMetaVersion100) &&
|
||||
format == fsMetaFormat)
|
||||
}
|
||||
|
||||
// Converts metadata to object info.
|
||||
func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo {
|
||||
if len(m.Meta) == 0 {
|
||||
@@ -66,7 +96,7 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
|
||||
Name: object,
|
||||
}
|
||||
|
||||
// We set file into only if its valid.
|
||||
// We set file info only if its valid.
|
||||
objInfo.ModTime = timeSentinel
|
||||
if fi != nil {
|
||||
objInfo.ModTime = fi.ModTime()
|
||||
@@ -74,17 +104,15 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
|
||||
objInfo.IsDir = fi.IsDir()
|
||||
}
|
||||
|
||||
objInfo.MD5Sum = m.Meta["md5Sum"]
|
||||
// Extract etag from metadata.
|
||||
objInfo.ETag = extractETag(m.Meta)
|
||||
objInfo.ContentType = m.Meta["content-type"]
|
||||
objInfo.ContentEncoding = m.Meta["content-encoding"]
|
||||
|
||||
// md5Sum has already been extracted into objInfo.MD5Sum. We
|
||||
// need to remove it from m.Meta to avoid it from appearing as
|
||||
// part of response headers. e.g, X-Minio-* or X-Amz-*.
|
||||
delete(m.Meta, "md5Sum")
|
||||
|
||||
// Save all the other userdefined API.
|
||||
objInfo.UserDefined = m.Meta
|
||||
// etag/md5Sum has already been extracted. We need to
|
||||
// remove to avoid it from appearing as part of
|
||||
// response headers. e.g, X-Minio-* or X-Amz-*.
|
||||
objInfo.UserDefined = cleanMetaETag(m.Meta)
|
||||
|
||||
// Success..
|
||||
return objInfo
|
||||
@@ -144,42 +172,84 @@ func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
|
||||
return int64(len(metadataBytes)), nil
|
||||
}
|
||||
|
||||
func parseFSVersion(fsMetaBuf []byte) string {
|
||||
return gjson.GetBytes(fsMetaBuf, "version").String()
|
||||
}
|
||||
|
||||
func parseFSFormat(fsMetaBuf []byte) string {
|
||||
return gjson.GetBytes(fsMetaBuf, "format").String()
|
||||
}
|
||||
|
||||
func parseFSRelease(fsMetaBuf []byte) string {
|
||||
return gjson.GetBytes(fsMetaBuf, "minio.release").String()
|
||||
}
|
||||
|
||||
func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
|
||||
// Get xlMetaV1.Meta map.
|
||||
metaMapResult := gjson.GetBytes(fsMetaBuf, "meta").Map()
|
||||
metaMap := make(map[string]string)
|
||||
for key, valResult := range metaMapResult {
|
||||
metaMap[key] = valResult.String()
|
||||
}
|
||||
return metaMap
|
||||
}
|
||||
|
||||
func parseFSParts(fsMetaBuf []byte) []objectPartInfo {
|
||||
// Parse the FS Parts.
|
||||
partsResult := gjson.GetBytes(fsMetaBuf, "parts").Array()
|
||||
partInfo := make([]objectPartInfo, len(partsResult))
|
||||
for i, p := range partsResult {
|
||||
info := objectPartInfo{}
|
||||
info.Number = int(p.Get("number").Int())
|
||||
info.Name = p.Get("name").String()
|
||||
info.ETag = p.Get("etag").String()
|
||||
info.Size = p.Get("size").Int()
|
||||
partInfo[i] = info
|
||||
}
|
||||
return partInfo
|
||||
}
|
||||
|
||||
func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
|
||||
var metadataBytes []byte
|
||||
var fsMetaBuf []byte
|
||||
fi, err := lk.Stat()
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
|
||||
metadataBytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
||||
fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
|
||||
if len(metadataBytes) == 0 {
|
||||
if len(fsMetaBuf) == 0 {
|
||||
return 0, traceError(io.EOF)
|
||||
}
|
||||
|
||||
// Decode `fs.json` into fsMeta structure.
|
||||
if err = json.Unmarshal(metadataBytes, m); err != nil {
|
||||
return 0, traceError(err)
|
||||
// obtain version.
|
||||
m.Version = parseFSVersion(fsMetaBuf)
|
||||
|
||||
// obtain format.
|
||||
m.Format = parseFSFormat(fsMetaBuf)
|
||||
|
||||
// Verify if the format is valid, return corrupted format
|
||||
// for unrecognized formats.
|
||||
if !isFSMetaValid(m.Version, m.Format) {
|
||||
return 0, traceError(errCorruptedFormat)
|
||||
}
|
||||
|
||||
// obtain metadata.
|
||||
m.Meta = parseFSMetaMap(fsMetaBuf)
|
||||
|
||||
// obtain parts info list.
|
||||
m.Parts = parseFSParts(fsMetaBuf)
|
||||
|
||||
// obtain minio release date.
|
||||
m.Minio.Release = parseFSRelease(fsMetaBuf)
|
||||
|
||||
// Success.
|
||||
return int64(len(metadataBytes)), nil
|
||||
return int64(len(fsMetaBuf)), nil
|
||||
}
|
||||
|
||||
// FS metadata constants.
|
||||
const (
|
||||
// FS backend meta version.
|
||||
fsMetaVersion = "1.0.0"
|
||||
|
||||
// FS backend meta format.
|
||||
fsMetaFormat = "fs"
|
||||
|
||||
// Add more constants here.
|
||||
)
|
||||
|
||||
// newFSMetaV1 - initializes new fsMetaV1.
|
||||
func newFSMetaV1() (fsMeta fsMetaV1) {
|
||||
fsMeta = fsMetaV1{}
|
||||
@@ -189,60 +259,97 @@ func newFSMetaV1() (fsMeta fsMetaV1) {
|
||||
return fsMeta
|
||||
}
|
||||
|
||||
// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
|
||||
func newFSFormatV1() (format *formatConfigV1) {
|
||||
return &formatConfigV1{
|
||||
Version: "1",
|
||||
Format: "fs",
|
||||
FS: &fsFormat{
|
||||
Version: "1",
|
||||
},
|
||||
}
|
||||
}
|
||||
// Check if disk has already a valid format, holds a read lock and
|
||||
// upon success returns it to the caller to be closed.
|
||||
func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
|
||||
fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
|
||||
|
||||
// loads format.json from minioMetaBucket if it exists.
|
||||
func loadFormatFS(fsPath string) (*formatConfigV1, error) {
|
||||
rlk, err := lock.RLockedOpenFile(pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile))
|
||||
rlk, err := lock.RLockedOpenFile(preparePath(fsFormatPath))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, errUnformattedDisk
|
||||
// If format.json not found then
|
||||
// its an unformatted disk.
|
||||
return nil, traceError(errUnformattedDisk)
|
||||
}
|
||||
return nil, err
|
||||
return nil, traceError(err)
|
||||
}
|
||||
defer rlk.Close()
|
||||
|
||||
formatBytes, err := ioutil.ReadAll(rlk)
|
||||
if err != nil {
|
||||
var format = &formatConfigV1{}
|
||||
if err = format.LoadFormat(rlk.LockedFile); err != nil {
|
||||
rlk.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
format := &formatConfigV1{}
|
||||
if err = json.Unmarshal(formatBytes, format); err != nil {
|
||||
// Check format FS.
|
||||
if err = format.CheckFS(); err != nil {
|
||||
rlk.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return format, nil
|
||||
// Always return read lock here and should be closed by the caller.
|
||||
return rlk, traceError(err)
|
||||
}
|
||||
|
||||
// writes FS format (format.json) into minioMetaBucket.
|
||||
func saveFormatFS(formatPath string, fsFormat *formatConfigV1) error {
|
||||
metadataBytes, err := json.Marshal(fsFormat)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Creates a new format.json if unformatted.
|
||||
func createFormatFS(fsPath string) error {
|
||||
fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
|
||||
|
||||
// fsFormatJSONFile - format.json file stored in minioMetaBucket(.minio.sys) directory.
|
||||
lk, err := lock.LockedOpenFile(preparePath(formatPath), os.O_CREATE|os.O_WRONLY, 0600)
|
||||
// Attempt a write lock on formatConfigFile `format.json`
|
||||
// file stored in minioMetaBucket(.minio.sys) directory.
|
||||
lk, err := lock.TryLockedOpenFile(preparePath(fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return err
|
||||
return traceError(err)
|
||||
}
|
||||
// Close the locked file upon return.
|
||||
defer lk.Close()
|
||||
|
||||
_, err = lk.Write(metadataBytes)
|
||||
// Success.
|
||||
// Load format on disk, checks if we are unformatted
|
||||
// writes the new format.json
|
||||
var format = &formatConfigV1{}
|
||||
err = format.LoadFormat(lk)
|
||||
if errorCause(err) == errUnformattedDisk {
|
||||
_, err = newFSFormat().WriteTo(lk)
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
|
||||
// This loop validates format.json by holding a read lock and
|
||||
// proceeds if disk unformatted to hold non-blocking WriteLock
|
||||
// If for some reason non-blocking WriteLock fails and the error
|
||||
// is lock.ErrAlreadyLocked i.e some other process is holding a
|
||||
// lock we retry in the loop again.
|
||||
for {
|
||||
// Validate the `format.json` for expected values.
|
||||
rlk, err = checkLockedValidFormatFS(fsPath)
|
||||
switch {
|
||||
case err == nil:
|
||||
// Holding a read lock ensures that any write lock operation
|
||||
// is blocked if attempted in-turn avoiding corruption on
|
||||
// the backend disk.
|
||||
return rlk, nil
|
||||
case errorCause(err) == errUnformattedDisk:
|
||||
if err = createFormatFS(fsPath); err != nil {
|
||||
// Existing write locks detected.
|
||||
if errorCause(err) == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
|
||||
// Unexpected error, return.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Loop will continue to attempt a read-lock on `format.json`.
|
||||
default:
|
||||
// Unhandled error return.
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return if the part info in uploadedParts and completeParts are same.
|
||||
func isPartsSame(uploadedParts []objectPartInfo, completeParts []completePart) bool {
|
||||
if len(uploadedParts) != len(completeParts) {
|
||||
|
||||
@@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
@@ -49,7 +48,7 @@ func TestReadFSMetadata(t *testing.T) {
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
sha256sum := ""
|
||||
@@ -59,7 +58,7 @@ func TestReadFSMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
// Construct the full path of fs.json
|
||||
fsPath := pathJoin("buckets", bucketName, objectName, "fs.json")
|
||||
fsPath := pathJoin(bucketMetaPrefix, bucketName, objectName, "fs.json")
|
||||
fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath)
|
||||
|
||||
rlk, err := fs.rwPool.Open(fsPath)
|
||||
@@ -73,18 +72,6 @@ func TestReadFSMetadata(t *testing.T) {
|
||||
if _, err = fsMeta.ReadFrom(rlk.LockedFile); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
// Corrupted fs.json
|
||||
file, err := os.OpenFile(preparePath(fsPath), os.O_APPEND|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
file.Write([]byte{'a'})
|
||||
file.Close()
|
||||
fsMeta = fsMetaV1{}
|
||||
if _, err := fsMeta.ReadFrom(rlk.LockedFile); err == nil {
|
||||
t.Fatal("Should fail", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteFSMetadata - tests of writeFSMetadata with healthy disk.
|
||||
@@ -98,7 +85,7 @@ func TestWriteFSMetadata(t *testing.T) {
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
sha256sum := ""
|
||||
@@ -108,7 +95,7 @@ func TestWriteFSMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
// Construct the full path of fs.json
|
||||
fsPath := pathJoin("buckets", bucketName, objectName, "fs.json")
|
||||
fsPath := pathJoin(bucketMetaPrefix, bucketName, objectName, "fs.json")
|
||||
fsPath = pathJoin(fs.fsPath, minioMetaBucket, fsPath)
|
||||
|
||||
rlk, err := fs.rwPool.Open(fsPath)
|
||||
@@ -123,10 +110,10 @@ func TestWriteFSMetadata(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
if fsMeta.Version != "1.0.0" {
|
||||
if fsMeta.Version != fsMetaVersion {
|
||||
t.Fatalf("Unexpected version %s", fsMeta.Version)
|
||||
}
|
||||
if fsMeta.Format != "fs" {
|
||||
if fsMeta.Format != fsMetaFormat {
|
||||
t.Fatalf("Unexpected format %s", fsMeta.Format)
|
||||
}
|
||||
}
|
||||
|
||||
+11
-48
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -24,7 +24,6 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
pathutil "path"
|
||||
"runtime"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -46,56 +45,15 @@ func (fs fsObjects) isMultipartUpload(bucket, prefix string) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// Delete uploads.json file wrapper handling a tricky case on windows.
|
||||
// Delete uploads.json file wrapper
|
||||
func (fs fsObjects) deleteUploadsJSON(bucket, object, uploadID string) error {
|
||||
timeID := fmt.Sprintf("%X", time.Now().UTC().UnixNano())
|
||||
tmpPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID+"+"+timeID)
|
||||
|
||||
multipartBucketPath := pathJoin(fs.fsPath, minioMetaMultipartBucket)
|
||||
uploadPath := pathJoin(multipartBucketPath, bucket, object)
|
||||
uploadsMetaPath := pathJoin(uploadPath, uploadsJSONFile)
|
||||
|
||||
// Special case for windows please read through.
|
||||
if runtime.GOOS == globalWindowsOSName {
|
||||
// Ordinarily windows does not permit deletion or renaming of files still
|
||||
// in use, but if all open handles to that file were opened with FILE_SHARE_DELETE
|
||||
// then it can permit renames and deletions of open files.
|
||||
//
|
||||
// There are however some gotchas with this, and it is worth listing them here.
|
||||
// Firstly, Windows never allows you to really delete an open file, rather it is
|
||||
// flagged as delete pending and its entry in its directory remains visible
|
||||
// (though no new file handles may be opened to it) and when the very last
|
||||
// open handle to the file in the system is closed, only then is it truly
|
||||
// deleted. Well, actually only sort of truly deleted, because Windows only
|
||||
// appears to remove the file entry from the directory, but in fact that
|
||||
// entry is merely hidden and actually still exists and attempting to create
|
||||
// a file with the same name will return an access denied error. How long it
|
||||
// silently exists for depends on a range of factors, but put it this way:
|
||||
// if your code loops creating and deleting the same file name as you might
|
||||
// when operating a lock file, you're going to see lots of random spurious
|
||||
// access denied errors and truly dismal lock file performance compared to POSIX.
|
||||
//
|
||||
// We work-around these un-POSIX file semantics by taking a dual step to
|
||||
// deleting files. Firstly, it renames the file to tmp location into multipartTmpBucket
|
||||
// We always open files with FILE_SHARE_DELETE permission enabled, with that
|
||||
// flag Windows permits renaming and deletion, and because the name was changed
|
||||
// to a very random name somewhere not in its origin directory before deletion,
|
||||
// you don't see those unexpected random errors when creating files with the
|
||||
// same name as a recently deleted file as you do anywhere else on Windows.
|
||||
// Because the file is probably not in its original containing directory any more,
|
||||
// deletions of that directory will not fail with "directory not empty" as they
|
||||
// otherwise normally would either.
|
||||
fsRenameFile(uploadsMetaPath, tmpPath)
|
||||
tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
|
||||
|
||||
// Proceed to deleting the directory.
|
||||
if err := fsDeleteFile(multipartBucketPath, uploadPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Finally delete the renamed file.
|
||||
return fsDeleteFile(pathutil.Dir(tmpPath), tmpPath)
|
||||
}
|
||||
return fsDeleteFile(multipartBucketPath, uploadsMetaPath)
|
||||
return fsRemoveMeta(multipartBucketPath, uploadsMetaPath, tmpDir)
|
||||
}
|
||||
|
||||
// Removes the uploadID, called either by CompleteMultipart of AbortMultipart. If the resuling uploads
|
||||
@@ -380,7 +338,7 @@ func (fs fsObjects) newMultipartUpload(bucket string, object string, meta map[st
|
||||
fsMeta.Meta = meta
|
||||
|
||||
uploadID = mustGetUUID()
|
||||
initiated := time.Now().UTC()
|
||||
initiated := UTCNow()
|
||||
|
||||
// Add upload ID to uploads.json
|
||||
uploadsPath := pathJoin(bucket, object, uploadsJSONFile)
|
||||
@@ -748,6 +706,11 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
// Check if an object is present as one of the parent dir.
|
||||
if fs.parentDirIsObject(bucket, pathutil.Dir(object)) {
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
|
||||
}
|
||||
|
||||
if _, err := fs.statBucketDir(bucket); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
@@ -908,7 +871,7 @@ func (fs fsObjects) CompleteMultipartUpload(bucket string, object string, upload
|
||||
if len(fsMeta.Meta) == 0 {
|
||||
fsMeta.Meta = make(map[string]string)
|
||||
}
|
||||
fsMeta.Meta["md5Sum"] = s3MD5
|
||||
fsMeta.Meta["etag"] = s3MD5
|
||||
|
||||
// Write all the set metadata.
|
||||
if _, err = fsMeta.WriteTo(metaFile); err != nil {
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestFSWriteUploadJSON(t *testing.T) {
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
|
||||
obj.MakeBucket(bucketName)
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
_, err := obj.NewMultipartUpload(bucketName, objectName, nil)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
@@ -60,7 +60,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Cannot create bucket, err: ", err)
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
|
||||
data := []byte("12345")
|
||||
dataLen := int64(len(data))
|
||||
|
||||
if err = obj.MakeBucket(bucketName); err != nil {
|
||||
if err = obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Cannot create bucket, err: ", err)
|
||||
}
|
||||
|
||||
@@ -122,7 +122,7 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
|
||||
objectName := "object"
|
||||
data := []byte("12345")
|
||||
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Cannot create bucket, err: ", err)
|
||||
}
|
||||
|
||||
@@ -161,7 +161,7 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
|
||||
objectName := "object"
|
||||
data := []byte("12345")
|
||||
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Cannot create bucket, err: ", err)
|
||||
}
|
||||
|
||||
|
||||
+131
-42
@@ -22,7 +22,9 @@ import (
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"syscall"
|
||||
@@ -40,6 +42,9 @@ type fsObjects struct {
|
||||
// temporary transactions.
|
||||
fsUUID string
|
||||
|
||||
// This value shouldn't be touched, once initialized.
|
||||
fsFormatRlk *lock.RLockedFile // Is a read lock on `format.json`.
|
||||
|
||||
// FS rw pool.
|
||||
rwPool *fsIOPool
|
||||
|
||||
@@ -84,7 +89,7 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fi, err := os.Stat(preparePath(fsPath))
|
||||
fi, err := osStat(preparePath(fsPath))
|
||||
if err == nil {
|
||||
if !fi.IsDir() {
|
||||
return nil, syscall.ENOTDIR
|
||||
@@ -107,24 +112,10 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
return nil, fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
|
||||
}
|
||||
|
||||
// Load `format.json`.
|
||||
format, err := loadFormatFS(fsPath)
|
||||
if err != nil && err != errUnformattedDisk {
|
||||
return nil, fmt.Errorf("Unable to load 'format.json', %s", err)
|
||||
}
|
||||
|
||||
// If the `format.json` doesn't exist create one.
|
||||
if err == errUnformattedDisk {
|
||||
fsFormatPath := pathJoin(fsPath, minioMetaBucket, fsFormatJSONFile)
|
||||
// Initialize format.json, if already exists overwrite it.
|
||||
if serr := saveFormatFS(fsFormatPath, newFSFormatV1()); serr != nil {
|
||||
return nil, fmt.Errorf("Unable to initialize 'format.json', %s", serr)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate if we have the same format.
|
||||
if err == nil && format.Format != "fs" {
|
||||
return nil, fmt.Errorf("Unable to recognize backend format, Disk is not in FS format. %s", format.Format)
|
||||
// Initialize `format.json`, this function also returns.
|
||||
rlk, err := initFormatFS(fsPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Initialize fs objects.
|
||||
@@ -140,6 +131,12 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
},
|
||||
}
|
||||
|
||||
// Once the filesystem has initialized hold the read lock for
|
||||
// the life time of the server. This is done to ensure that under
|
||||
// shared backend mode for FS, remote servers do not migrate
|
||||
// or cause changes on backend format.
|
||||
fs.fsFormatRlk = rlk
|
||||
|
||||
// Initialize and load bucket policies.
|
||||
err = initBucketPolicies(fs)
|
||||
if err != nil {
|
||||
@@ -203,7 +200,7 @@ func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
|
||||
|
||||
// MakeBucket - create a new bucket, returns if it
|
||||
// already exists.
|
||||
func (fs fsObjects) MakeBucket(bucket string) error {
|
||||
func (fs fsObjects) MakeBucketWithLocation(bucket, location string) error {
|
||||
bucketDir, err := fs.getBucketDir(bucket)
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
@@ -223,7 +220,7 @@ func (fs fsObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
|
||||
return BucketInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
// As os.Stat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
|
||||
// As osStat() doesn't carry other than ModTime(), use ModTime() as CreatedTime.
|
||||
createdTime := st.ModTime()
|
||||
return BucketInfo{
|
||||
Name: bucket,
|
||||
@@ -262,7 +259,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
|
||||
|
||||
bucketInfos = append(bucketInfos, BucketInfo{
|
||||
Name: fi.Name(),
|
||||
// As os.Stat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
|
||||
// As osStat() doesnt carry CreatedTime, use ModTime() as CreatedTime.
|
||||
Created: fi.ModTime(),
|
||||
})
|
||||
}
|
||||
@@ -470,12 +467,6 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (ObjectInfo, error) {
|
||||
|
||||
// GetObjectInfo - reads object metadata and replies back ObjectInfo.
|
||||
func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
|
||||
// This is a special case with object whose name ends with
|
||||
// a slash separator, we always return object not found here.
|
||||
if hasSuffix(object, slashSeparator) {
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileNotFound), bucket, object)
|
||||
}
|
||||
|
||||
if err := checkGetObjArgs(bucket, object); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
@@ -487,23 +478,55 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (ObjectInfo, error) {
|
||||
return fs.getObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
// This function does the following check, suppose
|
||||
// object is "a/b/c/d", stat makes sure that objects ""a/b/c""
|
||||
// "a/b" and "a" do not exist.
|
||||
func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
|
||||
var isParentDirObject func(string) bool
|
||||
isParentDirObject = func(p string) bool {
|
||||
if p == "." {
|
||||
return false
|
||||
}
|
||||
if _, err := fsStatFile(pathJoin(fs.fsPath, bucket, p)); err == nil {
|
||||
// If there is already a file at prefix "p" return error.
|
||||
return true
|
||||
}
|
||||
// Check if there is a file as one of the parent paths.
|
||||
return isParentDirObject(path.Dir(p))
|
||||
}
|
||||
return isParentDirObject(parent)
|
||||
}
|
||||
|
||||
// PutObject - creates an object upon reading from the input stream
|
||||
// until EOF, writes data directly to configured filesystem path.
|
||||
// Additionally writes `fs.json` which carries the necessary metadata
|
||||
// for future object operations.
|
||||
func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
|
||||
// This is a special case with size as '0' and object ends with
|
||||
// a slash separator, we treat it like a valid operation and
|
||||
// return success.
|
||||
func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, retErr error) {
|
||||
var err error
|
||||
|
||||
// Validate if bucket name is valid and exists.
|
||||
if _, err = fs.statBucketDir(bucket); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
// This is a special case with size as '0' and object ends
|
||||
// with a slash separator, we treat it like a valid operation
|
||||
// and return success.
|
||||
if isObjectDir(object, size) {
|
||||
// Check if an object is present as one of the parent dir.
|
||||
if fs.parentDirIsObject(bucket, path.Dir(object)) {
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
|
||||
}
|
||||
return dirObjectInfo(bucket, object, size, metadata), nil
|
||||
}
|
||||
|
||||
if err = checkPutObjectArgs(bucket, object, fs); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
if _, err = fs.statBucketDir(bucket); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket)
|
||||
// Check if an object is present as one of the parent dir.
|
||||
if fs.parentDirIsObject(bucket, path.Dir(object)) {
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
|
||||
}
|
||||
|
||||
// No metadata is set, allocate a new one.
|
||||
@@ -516,13 +539,21 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
|
||||
|
||||
var wlk *lock.LockedFile
|
||||
if bucket != minioMetaBucket {
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
bucketMetaDir := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix)
|
||||
fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fsMetaJSONFile)
|
||||
wlk, err = fs.rwPool.Create(fsMetaPath)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
|
||||
}
|
||||
// This close will allow for locks to be synchronized on `fs.json`.
|
||||
defer wlk.Close()
|
||||
defer func() {
|
||||
// Remove meta file when PutObject encounters any error
|
||||
if retErr != nil {
|
||||
tmpDir := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID)
|
||||
fsRemoveMeta(bucketMetaDir, fsMetaPath, tmpDir)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Uploaded object will first be written to the temporary location which will eventually
|
||||
@@ -581,12 +612,12 @@ func (fs fsObjects) PutObject(bucket string, object string, size int64, data io.
|
||||
|
||||
newMD5Hex := hex.EncodeToString(md5Writer.Sum(nil))
|
||||
// Update the md5sum if not set with the newly calculated one.
|
||||
if len(metadata["md5Sum"]) == 0 {
|
||||
metadata["md5Sum"] = newMD5Hex
|
||||
if len(metadata["etag"]) == 0 {
|
||||
metadata["etag"] = newMD5Hex
|
||||
}
|
||||
|
||||
// md5Hex representation.
|
||||
md5Hex := metadata["md5Sum"]
|
||||
md5Hex := metadata["etag"]
|
||||
if md5Hex != "" {
|
||||
if newMD5Hex != md5Hex {
|
||||
// Returns md5 mismatch.
|
||||
@@ -687,6 +718,45 @@ func (fs fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
|
||||
return listDir
|
||||
}
|
||||
|
||||
// getObjectETag is a helper function, which returns only the md5sum
|
||||
// of the file on the disk.
|
||||
func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fsMetaJSONFile)
|
||||
|
||||
// Read `fs.json` to perhaps contend with
|
||||
// parallel Put() operations.
|
||||
rlk, err := fs.rwPool.Open(fsMetaPath)
|
||||
// Ignore if `fs.json` is not available, this is true for pre-existing data.
|
||||
if err != nil && err != errFileNotFound {
|
||||
return "", toObjectErr(traceError(err), bucket, entry)
|
||||
}
|
||||
|
||||
// If file is not found, we don't need to proceed forward.
|
||||
if err == errFileNotFound {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
// Read from fs metadata only if it exists.
|
||||
defer fs.rwPool.Close(fsMetaPath)
|
||||
|
||||
fsMetaBuf, err := ioutil.ReadAll(rlk.LockedFile)
|
||||
if err != nil {
|
||||
// `fs.json` can be empty due to previously failed
|
||||
// PutObject() transaction, if we arrive at such
|
||||
// a situation we just ignore and continue.
|
||||
if errorCause(err) != io.EOF {
|
||||
return "", toObjectErr(err, bucket, entry)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if FS metadata is valid, if not return error.
|
||||
if !isFSMetaValid(parseFSVersion(fsMetaBuf), parseFSFormat(fsMetaBuf)) {
|
||||
return "", toObjectErr(traceError(errCorruptedFormat), bucket, entry)
|
||||
}
|
||||
|
||||
return extractETag(parseFSMetaMap(fsMetaBuf)), nil
|
||||
}
|
||||
|
||||
// ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
|
||||
// state for future re-entrant list requests.
|
||||
func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
@@ -731,14 +801,33 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
|
||||
objInfo.IsDir = true
|
||||
return
|
||||
}
|
||||
|
||||
// Protect reading `fs.json`.
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, entry)
|
||||
objectLock.RLock()
|
||||
var etag string
|
||||
etag, err = fs.getObjectETag(bucket, entry)
|
||||
objectLock.RUnlock()
|
||||
if err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
// Stat the file to get file size.
|
||||
var fi os.FileInfo
|
||||
fi, err = fsStatFile(pathJoin(fs.fsPath, bucket, entry))
|
||||
if err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, entry)
|
||||
}
|
||||
fsMeta := fsMetaV1{}
|
||||
return fsMeta.ToObjectInfo(bucket, entry, fi), nil
|
||||
|
||||
// Success.
|
||||
return ObjectInfo{
|
||||
Name: entry,
|
||||
Bucket: bucket,
|
||||
Size: fi.Size(),
|
||||
ModTime: fi.ModTime(),
|
||||
IsDir: fi.IsDir(),
|
||||
ETag: etag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
heal := false // true only for xl.ListObjectsHeal()
|
||||
@@ -809,8 +898,8 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
|
||||
}
|
||||
|
||||
// HealObject - no-op for fs. Valid only for XL.
|
||||
func (fs fsObjects) HealObject(bucket, object string) error {
|
||||
return traceError(NotImplemented{})
|
||||
func (fs fsObjects) HealObject(bucket, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealBucket - no-op for fs, Valid only for XL.
|
||||
|
||||
+75
-48
@@ -63,7 +63,7 @@ func TestFSShutdown(t *testing.T) {
|
||||
obj := initFSObjects(disk, t)
|
||||
fs := obj.(*fsObjects)
|
||||
objectContent := "12345"
|
||||
obj.MakeBucket(bucketName)
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
sha256sum := ""
|
||||
obj.PutObject(bucketName, objectName, int64(len(objectContent)), bytes.NewReader([]byte(objectContent)), nil, sha256sum)
|
||||
return fs, disk
|
||||
@@ -85,47 +85,6 @@ func TestFSShutdown(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSLoadFormatFS - test loadFormatFS with healty and faulty disks
|
||||
func TestFSLoadFormatFS(t *testing.T) {
|
||||
// Prepare for testing
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer removeAll(disk)
|
||||
|
||||
// Assign a new UUID.
|
||||
uuid := mustGetUUID()
|
||||
|
||||
// Initialize meta volume, if volume already exists ignores it.
|
||||
if err := initMetaVolumeFS(disk, uuid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fsFormatPath := pathJoin(disk, minioMetaBucket, fsFormatJSONFile)
|
||||
if err := saveFormatFS(preparePath(fsFormatPath), newFSFormatV1()); err != nil {
|
||||
t.Fatal("Should not fail here", err)
|
||||
}
|
||||
_, err := loadFormatFS(disk)
|
||||
if err != nil {
|
||||
t.Fatal("Should not fail here", err)
|
||||
}
|
||||
// Loading corrupted format file
|
||||
file, err := os.OpenFile(preparePath(fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
t.Fatal("Should not fail here", err)
|
||||
}
|
||||
file.Write([]byte{'b'})
|
||||
file.Close()
|
||||
_, err = loadFormatFS(disk)
|
||||
if err == nil {
|
||||
t.Fatal("Should return an error here")
|
||||
}
|
||||
// Loading format file from disk not found.
|
||||
removeAll(disk)
|
||||
_, err = loadFormatFS(disk)
|
||||
if err != nil && err != errUnformattedDisk {
|
||||
t.Fatal("Should return unformatted disk, but got", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSGetBucketInfo - test GetBucketInfo with healty and faulty disks
|
||||
func TestFSGetBucketInfo(t *testing.T) {
|
||||
// Prepare for testing
|
||||
@@ -136,7 +95,7 @@ func TestFSGetBucketInfo(t *testing.T) {
|
||||
fs := obj.(*fsObjects)
|
||||
bucketName := "bucket"
|
||||
|
||||
obj.MakeBucket(bucketName)
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
|
||||
// Test with valid parameters
|
||||
info, err := fs.GetBucketInfo(bucketName)
|
||||
@@ -161,6 +120,74 @@ func TestFSGetBucketInfo(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSPutObject(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer removeAll(disk)
|
||||
|
||||
obj := initFSObjects(disk, t)
|
||||
bucketName := "bucket"
|
||||
objectName := "1/2/3/4/object"
|
||||
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
sha256sum := ""
|
||||
|
||||
// With a regular object.
|
||||
_, err := obj.PutObject(bucketName+"non-existent", objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
if _, ok := errorCause(err).(BucketNotFound); !ok {
|
||||
t.Fatalf("Expected error type BucketNotFound, got %#v", err)
|
||||
}
|
||||
|
||||
// With a directory object.
|
||||
_, err = obj.PutObject(bucketName+"non-existent", objectName+"/", int64(0), bytes.NewReader([]byte("")), nil, sha256sum)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
if _, ok := errorCause(err).(BucketNotFound); !ok {
|
||||
t.Fatalf("Expected error type BucketNotFound, got %#v", err)
|
||||
}
|
||||
|
||||
_, err = obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = obj.PutObject(bucketName, objectName+"/1", int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backend corruption occurred")
|
||||
}
|
||||
if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
|
||||
t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
|
||||
} else {
|
||||
if nerr.Bucket != "bucket" {
|
||||
t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
|
||||
}
|
||||
if nerr.Object != "1/2/3/4/object/1" {
|
||||
t.Fatalf("Expected '1/2/3/4/object/1', got %s", nerr.Object)
|
||||
}
|
||||
}
|
||||
|
||||
_, err = obj.PutObject(bucketName, objectName+"/1/", 0, bytes.NewReader([]byte("")), nil, sha256sum)
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backned corruption occurred")
|
||||
}
|
||||
if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
|
||||
t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
|
||||
} else {
|
||||
if nerr.Bucket != "bucket" {
|
||||
t.Fatalf("Expected 'bucket', got %s", nerr.Bucket)
|
||||
}
|
||||
if nerr.Object != "1/2/3/4/object/1/" {
|
||||
t.Fatalf("Expected '1/2/3/4/object/1/', got %s", nerr.Object)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSDeleteObject - test fs.DeleteObject() with healthy and corrupted disks
|
||||
func TestFSDeleteObject(t *testing.T) {
|
||||
// Prepare for tests
|
||||
@@ -172,7 +199,7 @@ func TestFSDeleteObject(t *testing.T) {
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
|
||||
obj.MakeBucket(bucketName)
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
sha256sum := ""
|
||||
obj.PutObject(bucketName, objectName, int64(len("abcd")), bytes.NewReader([]byte("abcd")), nil, sha256sum)
|
||||
|
||||
@@ -217,7 +244,7 @@ func TestFSDeleteBucket(t *testing.T) {
|
||||
fs := obj.(*fsObjects)
|
||||
bucketName := "bucket"
|
||||
|
||||
err := obj.MakeBucket(bucketName)
|
||||
err := obj.MakeBucketWithLocation(bucketName, "")
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
@@ -235,7 +262,7 @@ func TestFSDeleteBucket(t *testing.T) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
obj.MakeBucket(bucketName)
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
|
||||
// Delete bucker should get error disk not found.
|
||||
removeAll(disk)
|
||||
@@ -256,7 +283,7 @@ func TestFSListBuckets(t *testing.T) {
|
||||
fs := obj.(*fsObjects)
|
||||
|
||||
bucketName := "bucket"
|
||||
if err := obj.MakeBucket(bucketName); err != nil {
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
@@ -303,7 +330,7 @@ func TestFSHealObject(t *testing.T) {
|
||||
defer removeAll(disk)
|
||||
|
||||
obj := initFSObjects(disk, t)
|
||||
err := obj.HealObject("bucket", "object")
|
||||
_, _, err := obj.HealObject("bucket", "object")
|
||||
if err == nil || !isSameType(errorCause(err), NotImplemented{}) {
|
||||
t.Fatalf("Heal Object should return NotImplemented error ")
|
||||
}
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
)
|
||||
|
||||
// AnonGetBucketInfo - Get bucket metadata from azure anonymously.
|
||||
func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
|
||||
func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
|
||||
url, err := url.Parse(a.client.GetBlobURL(bucket, ""))
|
||||
if err != nil {
|
||||
return bucketInfo, azureToObjectError(traceError(err))
|
||||
@@ -40,7 +40,7 @@ func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, e
|
||||
if err != nil {
|
||||
return bucketInfo, azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return bucketInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
|
||||
@@ -57,11 +57,18 @@ func (a AzureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, e
|
||||
return bucketInfo, nil
|
||||
}
|
||||
|
||||
// AnonPutObject - SendPUT request without authentication.
|
||||
// This is needed when clients send PUT requests on objects that can be uploaded without auth.
|
||||
func (a *azureObjects) AnonPutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
|
||||
// azure doesn't support anonymous put
|
||||
return ObjectInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetObject - SendGET request without authentication.
|
||||
// This is needed when clients send GET requests on objects that can be downloaded without auth.
|
||||
func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
|
||||
url := a.client.GetBlobURL(bucket, object)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
|
||||
u := a.client.GetBlobURL(bucket, object)
|
||||
req, err := http.NewRequest("GET", u, nil)
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
@@ -88,12 +95,12 @@ func (a AzureObjects) AnonGetObject(bucket, object string, startOffset int64, le
|
||||
|
||||
// AnonGetObjectInfo - Send HEAD request without authentication and convert the
|
||||
// result to ObjectInfo.
|
||||
func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
resp, err := http.Head(a.client.GetBlobURL(bucket, object))
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return objInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
|
||||
@@ -120,7 +127,7 @@ func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectIn
|
||||
objInfo.UserDefined["Content-Encoding"] = resp.Header.Get("Content-Encoding")
|
||||
}
|
||||
objInfo.UserDefined["Content-Type"] = resp.Header.Get("Content-Type")
|
||||
objInfo.MD5Sum = resp.Header.Get("Etag")
|
||||
objInfo.ETag = resp.Header.Get("Etag")
|
||||
objInfo.ModTime = t
|
||||
objInfo.Name = object
|
||||
objInfo.Size = contentLength
|
||||
@@ -128,7 +135,7 @@ func (a AzureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectIn
|
||||
}
|
||||
|
||||
// AnonListObjects - Use Azure equivalent ListBlobs.
|
||||
func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
params := storage.ListBlobsParameters{
|
||||
Prefix: prefix,
|
||||
Marker: marker,
|
||||
@@ -175,7 +182,7 @@ func (a AzureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
|
||||
Name: object.Name,
|
||||
ModTime: t,
|
||||
Size: object.Properties.ContentLength,
|
||||
MD5Sum: object.Properties.Etag,
|
||||
ETag: object.Properties.Etag,
|
||||
ContentType: object.Properties.ContentType,
|
||||
ContentEncoding: object.Properties.ContentEncoding,
|
||||
})
|
||||
@@ -17,27 +17,27 @@
|
||||
package cmd
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (a AzureObjects) HealBucket(bucket string) error {
|
||||
func (a *azureObjects) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (a AzureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
func (a *azureObjects) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return nil, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (a AzureObjects) HealObject(bucket, object string) error {
|
||||
return traceError(NotImplemented{})
|
||||
func (a *azureObjects) HealObject(bucket, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (a AzureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
func (a *azureObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
return ListObjectsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (a AzureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
|
||||
func (a *azureObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
|
||||
delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
return ListMultipartsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
@@ -17,6 +17,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
@@ -36,6 +37,36 @@ import (
|
||||
|
||||
const globalAzureAPIVersion = "2016-05-31"
|
||||
|
||||
// Canonicalize the metadata headers, without this azure-sdk calculates
|
||||
// incorrect signature. This attempt to canonicalize is to convert
|
||||
// any HTTP header which is of form say `accept-encoding` should be
|
||||
// converted to `Accept-Encoding` in its canonical form.
|
||||
// Also replaces X-Amz-Meta prefix with X-Ms-Meta as Azure expects user
|
||||
// defined metadata to have X-Ms-Meta prefix.
|
||||
func s3ToAzureHeaders(headers map[string]string) (newHeaders map[string]string) {
|
||||
newHeaders = make(map[string]string)
|
||||
for k, v := range headers {
|
||||
k = http.CanonicalHeaderKey(k)
|
||||
if strings.HasPrefix(k, "X-Amz-Meta") {
|
||||
k = strings.Replace(k, "X-Amz-Meta", "X-Ms-Meta", -1)
|
||||
}
|
||||
newHeaders[k] = v
|
||||
}
|
||||
return newHeaders
|
||||
}
|
||||
|
||||
// Prefix user metadata with "X-Amz-Meta-".
|
||||
// client.GetBlobMetadata() already strips "X-Ms-Meta-"
|
||||
func azureToS3Metadata(meta map[string]string) (newMeta map[string]string) {
|
||||
newMeta = make(map[string]string)
|
||||
|
||||
for k, v := range meta {
|
||||
k = "X-Amz-Meta-" + k
|
||||
newMeta[k] = v
|
||||
}
|
||||
return newMeta
|
||||
}
|
||||
|
||||
// To store metadata during NewMultipartUpload which will be used after
|
||||
// CompleteMultipartUpload to call SetBlobMetadata.
|
||||
type azureMultipartMetaInfo struct {
|
||||
@@ -64,8 +95,8 @@ func (a *azureMultipartMetaInfo) del(key string) {
|
||||
delete(a.meta, key)
|
||||
}
|
||||
|
||||
// AzureObjects - Implements Object layer for Azure blob storage.
|
||||
type AzureObjects struct {
|
||||
// azureObjects - Implements Object layer for Azure blob storage.
|
||||
type azureObjects struct {
|
||||
client storage.BlobStorageClient // Azure sdk client
|
||||
metaInfo azureMultipartMetaInfo
|
||||
}
|
||||
@@ -122,16 +153,16 @@ func azureToObjectError(err error, params ...string) error {
|
||||
return e
|
||||
}
|
||||
|
||||
// Inits azure blob storage client and returns AzureObjects.
|
||||
// Inits azure blob storage client and returns azureObjects.
|
||||
func newAzureLayer(endPoint string, account, key string, secure bool) (GatewayLayer, error) {
|
||||
if endPoint == "" {
|
||||
endPoint = storage.DefaultBaseURL
|
||||
}
|
||||
c, err := storage.NewClient(account, key, endPoint, globalAzureAPIVersion, secure)
|
||||
if err != nil {
|
||||
return AzureObjects{}, err
|
||||
return &azureObjects{}, err
|
||||
}
|
||||
return &AzureObjects{
|
||||
return &azureObjects{
|
||||
client: c.GetBlobService(),
|
||||
metaInfo: azureMultipartMetaInfo{
|
||||
meta: make(map[string]map[string]string),
|
||||
@@ -142,24 +173,24 @@ func newAzureLayer(endPoint string, account, key string, secure bool) (GatewayLa
|
||||
|
||||
// Shutdown - save any gateway metadata to disk
|
||||
// if necessary and reload upon next restart.
|
||||
func (a AzureObjects) Shutdown() error {
|
||||
func (a *azureObjects) Shutdown() error {
|
||||
// TODO
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageInfo - Not relevant to Azure backend.
|
||||
func (a AzureObjects) StorageInfo() StorageInfo {
|
||||
func (a *azureObjects) StorageInfo() StorageInfo {
|
||||
return StorageInfo{}
|
||||
}
|
||||
|
||||
// MakeBucket - Create a new container on azure backend.
|
||||
func (a AzureObjects) MakeBucket(bucket string) error {
|
||||
// MakeBucketWithLocation - Create a new container on azure backend.
|
||||
func (a *azureObjects) MakeBucketWithLocation(bucket, location string) error {
|
||||
err := a.client.CreateContainer(bucket, storage.ContainerAccessTypePrivate)
|
||||
return azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
// GetBucketInfo - Get bucket metadata..
|
||||
func (a AzureObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
|
||||
func (a *azureObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
|
||||
// Azure does not have an equivalent call, hence use ListContainers.
|
||||
resp, err := a.client.ListContainers(storage.ListContainersParameters{
|
||||
Prefix: bucket,
|
||||
@@ -182,7 +213,7 @@ func (a AzureObjects) GetBucketInfo(bucket string) (BucketInfo, error) {
|
||||
}
|
||||
|
||||
// ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers.
|
||||
func (a AzureObjects) ListBuckets() (buckets []BucketInfo, err error) {
|
||||
func (a *azureObjects) ListBuckets() (buckets []BucketInfo, err error) {
|
||||
resp, err := a.client.ListContainers(storage.ListContainersParameters{})
|
||||
if err != nil {
|
||||
return nil, azureToObjectError(traceError(err))
|
||||
@@ -201,13 +232,13 @@ func (a AzureObjects) ListBuckets() (buckets []BucketInfo, err error) {
|
||||
}
|
||||
|
||||
// DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer.
|
||||
func (a AzureObjects) DeleteBucket(bucket string) error {
|
||||
func (a *azureObjects) DeleteBucket(bucket string) error {
|
||||
return azureToObjectError(traceError(a.client.DeleteContainer(bucket)), bucket)
|
||||
}
|
||||
|
||||
// ListObjects - lists all blobs on azure with in a container filtered by prefix
|
||||
// and marker, uses Azure equivalent ListBlobs.
|
||||
func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
resp, err := a.client.ListBlobs(bucket, storage.ListBlobsParameters{
|
||||
Prefix: prefix,
|
||||
Marker: marker,
|
||||
@@ -229,7 +260,7 @@ func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxK
|
||||
Name: object.Name,
|
||||
ModTime: t,
|
||||
Size: object.Properties.ContentLength,
|
||||
MD5Sum: canonicalizeETag(object.Properties.Etag),
|
||||
ETag: canonicalizeETag(object.Properties.Etag),
|
||||
ContentType: object.Properties.ContentType,
|
||||
ContentEncoding: object.Properties.ContentEncoding,
|
||||
})
|
||||
@@ -244,7 +275,7 @@ func (a AzureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxK
|
||||
//
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (a AzureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
byteRange := fmt.Sprintf("%d-", startOffset)
|
||||
if length > 0 && startOffset > 0 {
|
||||
byteRange = fmt.Sprintf("%d-%d", startOffset, startOffset+length-1)
|
||||
@@ -267,7 +298,14 @@ func (a AzureObjects) GetObject(bucket, object string, startOffset int64, length
|
||||
|
||||
// GetObjectInfo - reads blob metadata properties and replies back ObjectInfo,
|
||||
// uses zure equivalent GetBlobProperties.
|
||||
func (a AzureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
blobMeta, err := a.client.GetBlobMetadata(bucket, object)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
meta := azureToS3Metadata(blobMeta)
|
||||
|
||||
prop, err := a.client.GetBlobProperties(bucket, object)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
@@ -276,55 +314,69 @@ func (a AzureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo,
|
||||
if err != nil {
|
||||
return objInfo, traceError(err)
|
||||
}
|
||||
|
||||
if prop.ContentEncoding != "" {
|
||||
meta["Content-Encoding"] = prop.ContentEncoding
|
||||
}
|
||||
meta["Content-Type"] = prop.ContentType
|
||||
|
||||
objInfo = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
UserDefined: make(map[string]string),
|
||||
MD5Sum: canonicalizeETag(prop.Etag),
|
||||
UserDefined: meta,
|
||||
ETag: canonicalizeETag(prop.Etag),
|
||||
ModTime: t,
|
||||
Name: object,
|
||||
Size: prop.ContentLength,
|
||||
}
|
||||
if prop.ContentEncoding != "" {
|
||||
objInfo.UserDefined["Content-Encoding"] = prop.ContentEncoding
|
||||
}
|
||||
objInfo.UserDefined["Content-Type"] = prop.ContentType
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// Canonicalize the metadata headers, without this azure-sdk calculates
|
||||
// incorrect signature. This attempt to canonicalize is to convert
|
||||
// any HTTP header which is of form say `accept-encoding` should be
|
||||
// converted to `Accept-Encoding` in its canonical form.
|
||||
func canonicalMetadata(metadata map[string]string) (canonical map[string]string) {
|
||||
canonical = make(map[string]string)
|
||||
for k, v := range metadata {
|
||||
canonical[http.CanonicalHeaderKey(k)] = v
|
||||
}
|
||||
return canonical
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// PutObject - Create a new blob with the incoming data,
|
||||
// uses Azure equivalent CreateBlockBlobFromReader.
|
||||
func (a AzureObjects) PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) PutObject(bucket, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (objInfo ObjectInfo, err error) {
|
||||
var sha256Writer hash.Hash
|
||||
var md5sumWriter hash.Hash
|
||||
|
||||
var writers []io.Writer
|
||||
|
||||
md5sum := metadata["etag"]
|
||||
delete(metadata, "etag")
|
||||
|
||||
teeReader := data
|
||||
|
||||
if sha256sum != "" {
|
||||
sha256Writer = sha256.New()
|
||||
teeReader = io.TeeReader(data, sha256Writer)
|
||||
writers = append(writers, sha256Writer)
|
||||
}
|
||||
|
||||
delete(metadata, "md5Sum")
|
||||
if md5sum != "" {
|
||||
md5sumWriter = md5.New()
|
||||
writers = append(writers, md5sumWriter)
|
||||
}
|
||||
|
||||
err = a.client.CreateBlockBlobFromReader(bucket, object, uint64(size), teeReader, canonicalMetadata(metadata))
|
||||
if len(writers) > 0 {
|
||||
teeReader = io.TeeReader(data, io.MultiWriter(writers...))
|
||||
}
|
||||
|
||||
err = a.client.CreateBlockBlobFromReader(bucket, object, uint64(size), teeReader, s3ToAzureHeaders(metadata))
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
if md5sum != "" {
|
||||
newMD5sum := hex.EncodeToString(md5sumWriter.Sum(nil))
|
||||
if newMD5sum != md5sum {
|
||||
a.client.DeleteBlob(bucket, object, nil)
|
||||
return ObjectInfo{}, azureToObjectError(traceError(BadDigest{md5sum, newMD5sum}))
|
||||
}
|
||||
}
|
||||
|
||||
if sha256sum != "" {
|
||||
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
|
||||
if newSHA256sum != sha256sum {
|
||||
a.client.DeleteBlob(bucket, object, nil)
|
||||
return ObjectInfo{}, traceError(SHA256Mismatch{})
|
||||
return ObjectInfo{}, azureToObjectError(traceError(SHA256Mismatch{}))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -333,7 +385,7 @@ func (a AzureObjects) PutObject(bucket, object string, size int64, data io.Reade
|
||||
|
||||
// CopyObject - Copies a blob from source container to destination container.
|
||||
// Uses Azure equivalent CopyBlob API.
|
||||
func (a AzureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
err = a.client.CopyBlob(destBucket, destObject, a.client.GetBlobURL(srcBucket, srcObject))
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
|
||||
@@ -343,7 +395,7 @@ func (a AzureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject st
|
||||
|
||||
// DeleteObject - Deletes a blob on azure container, uses Azure
|
||||
// equivalent DeleteBlob API.
|
||||
func (a AzureObjects) DeleteObject(bucket, object string) error {
|
||||
func (a *azureObjects) DeleteObject(bucket, object string) error {
|
||||
err := a.client.DeleteBlob(bucket, object, nil)
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
@@ -355,7 +407,7 @@ func (a AzureObjects) DeleteObject(bucket, object string) error {
|
||||
// FIXME: Full ListMultipartUploads is not supported yet. It is supported just enough to help our client libs to
|
||||
// support re-uploads. a.client.ListBlobs() can be made to return entries which include uncommitted blobs using
|
||||
// which we need to filter out the committed blobs to get the list of uncommitted blobs.
|
||||
func (a AzureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
func (a *azureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
result.MaxUploads = maxUploads
|
||||
result.Prefix = prefix
|
||||
result.Delimiter = delimiter
|
||||
@@ -366,12 +418,12 @@ func (a AzureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMa
|
||||
// NewMultipartUpload.
|
||||
return result, nil
|
||||
}
|
||||
result.Uploads = []uploadMetadata{{prefix, prefix, time.Now().UTC(), "", nil}}
|
||||
result.Uploads = []uploadMetadata{{prefix, prefix, UTCNow(), "", nil}}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// NewMultipartUpload - Use Azure equivalent CreateBlockBlob.
|
||||
func (a AzureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
func (a *azureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
// Azure doesn't return a unique upload ID and we use object name in place of it. Azure allows multiple uploads to
|
||||
// co-exist as long as the user keeps the blocks uploaded (in block blobs) unique amongst concurrent upload attempts.
|
||||
// Each concurrent client, keeps its own blockID list which it can commit.
|
||||
@@ -380,14 +432,14 @@ func (a AzureObjects) NewMultipartUpload(bucket, object string, metadata map[str
|
||||
// Store an empty map as a placeholder else ListObjectParts/PutObjectPart will not work properly.
|
||||
metadata = make(map[string]string)
|
||||
} else {
|
||||
metadata = canonicalMetadata(metadata)
|
||||
metadata = s3ToAzureHeaders(metadata)
|
||||
}
|
||||
a.metaInfo.set(uploadID, metadata)
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// CopyObjectPart - Not implemented.
|
||||
func (a AzureObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
func (a *azureObjects) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return info, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
@@ -415,39 +467,66 @@ func azureParseBlockID(blockID string) (int, string, error) {
|
||||
}
|
||||
|
||||
// PutObjectPart - Use Azure equivalent PutBlockWithLength.
|
||||
func (a AzureObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (info PartInfo, err error) {
|
||||
func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (info PartInfo, err error) {
|
||||
if meta := a.metaInfo.get(uploadID); meta == nil {
|
||||
return info, traceError(InvalidUploadID{})
|
||||
}
|
||||
var sha256Writer hash.Hash
|
||||
var md5sumWriter hash.Hash
|
||||
var etag string
|
||||
|
||||
var writers []io.Writer
|
||||
|
||||
if sha256sum != "" {
|
||||
sha256Writer = sha256.New()
|
||||
writers = append(writers, sha256Writer)
|
||||
}
|
||||
|
||||
teeReader := io.TeeReader(data, sha256Writer)
|
||||
if md5Hex != "" {
|
||||
md5sumWriter = md5.New()
|
||||
writers = append(writers, md5sumWriter)
|
||||
etag = md5Hex
|
||||
} else {
|
||||
// Generate random ETag.
|
||||
etag = getMD5Hash([]byte(mustGetUUID()))
|
||||
}
|
||||
|
||||
id := azureGetBlockID(partID, md5Hex)
|
||||
teeReader := data
|
||||
|
||||
if len(writers) > 0 {
|
||||
teeReader = io.TeeReader(data, io.MultiWriter(writers...))
|
||||
}
|
||||
|
||||
id := azureGetBlockID(partID, etag)
|
||||
err = a.client.PutBlockWithLength(bucket, object, id, uint64(size), teeReader, nil)
|
||||
if err != nil {
|
||||
return info, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
if md5Hex != "" {
|
||||
newMD5sum := hex.EncodeToString(md5sumWriter.Sum(nil))
|
||||
if newMD5sum != md5Hex {
|
||||
a.client.DeleteBlob(bucket, object, nil)
|
||||
return PartInfo{}, azureToObjectError(traceError(BadDigest{md5Hex, newMD5sum}))
|
||||
}
|
||||
}
|
||||
|
||||
if sha256sum != "" {
|
||||
newSHA256sum := hex.EncodeToString(sha256Writer.Sum(nil))
|
||||
if newSHA256sum != sha256sum {
|
||||
return PartInfo{}, traceError(SHA256Mismatch{})
|
||||
return PartInfo{}, azureToObjectError(traceError(SHA256Mismatch{}))
|
||||
}
|
||||
}
|
||||
|
||||
info.PartNumber = partID
|
||||
info.ETag = md5Hex
|
||||
info.LastModified = time.Now().UTC()
|
||||
info.ETag = etag
|
||||
info.LastModified = UTCNow()
|
||||
info.Size = size
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// ListObjectParts - Use Azure equivalent GetBlockList.
|
||||
func (a AzureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
|
||||
func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
|
||||
result.Bucket = bucket
|
||||
result.Object = object
|
||||
result.UploadID = uploadID
|
||||
@@ -478,7 +557,7 @@ func (a AzureObjects) ListObjectParts(bucket, object, uploadID string, partNumbe
|
||||
}
|
||||
result.Parts = append(result.Parts, PartInfo{
|
||||
partID,
|
||||
time.Now().UTC(),
|
||||
UTCNow(),
|
||||
md5Hex,
|
||||
part.Size,
|
||||
})
|
||||
@@ -496,13 +575,13 @@ func (a AzureObjects) ListObjectParts(bucket, object, uploadID string, partNumbe
|
||||
// AbortMultipartUpload - Not Implemented.
|
||||
// There is no corresponding API in azure to abort an incomplete upload. The uncommmitted blocks
|
||||
// gets deleted after one week.
|
||||
func (a AzureObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
|
||||
func (a *azureObjects) AbortMultipartUpload(bucket, object, uploadID string) error {
|
||||
a.metaInfo.del(uploadID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - Use Azure equivalent PutBlockList.
|
||||
func (a AzureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []completePart) (objInfo ObjectInfo, err error) {
|
||||
meta := a.metaInfo.get(uploadID)
|
||||
if meta == nil {
|
||||
return objInfo, traceError(InvalidUploadID{uploadID})
|
||||
@@ -530,6 +609,10 @@ func (a AzureObjects) CompleteMultipartUpload(bucket, object, uploadID string, u
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
err = a.client.SetBlobMetadata(bucket, object, nil, meta)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
}
|
||||
a.metaInfo.del(uploadID)
|
||||
return a.GetObjectInfo(bucket, object)
|
||||
@@ -592,7 +675,15 @@ func azureListBlobsGetParameters(p storage.ListBlobsParameters) url.Values {
|
||||
// storage.ContainerAccessTypePrivate - none in minio terminology
|
||||
// As the common denominator for minio and azure is readonly and none, we support
|
||||
// these two policies at the bucket level.
|
||||
func (a AzureObjects) SetBucketPolicies(bucket string, policies []BucketAccessPolicy) error {
|
||||
func (a *azureObjects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
|
||||
var policies []BucketAccessPolicy
|
||||
|
||||
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
|
||||
policies = append(policies, BucketAccessPolicy{
|
||||
Prefix: prefix,
|
||||
Policy: policy,
|
||||
})
|
||||
}
|
||||
prefix := bucket + "/*" // For all objects inside the bucket.
|
||||
if len(policies) != 1 {
|
||||
return traceError(NotImplemented{})
|
||||
@@ -612,22 +703,25 @@ func (a AzureObjects) SetBucketPolicies(bucket string, policies []BucketAccessPo
|
||||
}
|
||||
|
||||
// GetBucketPolicies - Get the container ACL and convert it to canonical []bucketAccessPolicy
|
||||
func (a AzureObjects) GetBucketPolicies(bucket string) ([]BucketAccessPolicy, error) {
|
||||
func (a *azureObjects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
|
||||
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
|
||||
perm, err := a.client.GetContainerPermissions(bucket, 0, "")
|
||||
if err != nil {
|
||||
return nil, azureToObjectError(traceError(err), bucket)
|
||||
return policy.BucketAccessPolicy{}, azureToObjectError(traceError(err), bucket)
|
||||
}
|
||||
switch perm.AccessType {
|
||||
case storage.ContainerAccessTypePrivate:
|
||||
return nil, nil
|
||||
// Do nothing
|
||||
case storage.ContainerAccessTypeContainer:
|
||||
return []BucketAccessPolicy{{"", policy.BucketPolicyReadOnly}}, nil
|
||||
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
|
||||
default:
|
||||
return policy.BucketAccessPolicy{}, azureToObjectError(traceError(NotImplemented{}))
|
||||
}
|
||||
return nil, azureToObjectError(traceError(NotImplemented{}))
|
||||
return policyInfo, nil
|
||||
}
|
||||
|
||||
// DeleteBucketPolicies - Set the container ACL to "private"
|
||||
func (a AzureObjects) DeleteBucketPolicies(bucket string) error {
|
||||
func (a *azureObjects) DeleteBucketPolicies(bucket string) error {
|
||||
perm := storage.ContainerPermissions{
|
||||
AccessType: storage.ContainerAccessTypePrivate,
|
||||
AccessPolicies: nil,
|
||||
@@ -25,18 +25,33 @@ import (
|
||||
)
|
||||
|
||||
// Test canonical metadata.
|
||||
func TestCanonicalMetadata(t *testing.T) {
|
||||
metadata := map[string]string{
|
||||
func TestS3ToAzureHeaders(t *testing.T) {
|
||||
headers := map[string]string{
|
||||
"accept-encoding": "gzip",
|
||||
"content-encoding": "gzip",
|
||||
}
|
||||
expectedCanonicalM := map[string]string{
|
||||
expectedHeaders := map[string]string{
|
||||
"Accept-Encoding": "gzip",
|
||||
"Content-Encoding": "gzip",
|
||||
}
|
||||
actualCanonicalM := canonicalMetadata(metadata)
|
||||
if !reflect.DeepEqual(actualCanonicalM, expectedCanonicalM) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedCanonicalM, actualCanonicalM)
|
||||
actualHeaders := s3ToAzureHeaders(headers)
|
||||
if !reflect.DeepEqual(actualHeaders, expectedHeaders) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedHeaders, actualHeaders)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAzureToS3Metadata(t *testing.T) {
|
||||
// Just one testcase. Adding more test cases does not add value to the testcase
|
||||
// as azureToS3Metadata() just adds a prefix.
|
||||
metadata := map[string]string{
|
||||
"First-Name": "myname",
|
||||
}
|
||||
expectedMeta := map[string]string{
|
||||
"X-Amz-Meta-First-Name": "myname",
|
||||
}
|
||||
actualMeta := azureToS3Metadata(metadata)
|
||||
if !reflect.DeepEqual(actualMeta, expectedMeta) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, actualMeta)
|
||||
}
|
||||
}
|
||||
|
||||
+240
-45
@@ -17,11 +17,12 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
|
||||
@@ -53,17 +54,23 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeAnonymous:
|
||||
// No verification needed for anonymous requests.
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
@@ -152,6 +159,138 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
}
|
||||
|
||||
// PutObjectHandler - PUT Object
|
||||
// ----------
|
||||
// This implementation of the PUT operation adds an object to a bucket.
|
||||
func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// X-Amz-Copy-Source shouldn't be set for this call.
|
||||
if _, ok := r.Header["X-Amz-Copy-Source"]; ok {
|
||||
writeErrorResponse(w, ErrInvalidCopySource, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var object, bucket string
|
||||
vars := router.Vars(r)
|
||||
bucket = vars["bucket"]
|
||||
object = vars["object"]
|
||||
|
||||
// Get Content-Md5 sent by client and verify if valid
|
||||
md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5"))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to validate content-md5 format.")
|
||||
writeErrorResponse(w, ErrInvalidDigest, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
/// if Content-Length is unknown/missing, deny the request
|
||||
size := r.ContentLength
|
||||
reqAuthType := getRequestAuthType(r)
|
||||
if reqAuthType == authTypeStreamingSigned {
|
||||
sizeStr := r.Header.Get("x-amz-decoded-content-length")
|
||||
size, err = strconv.ParseInt(sizeStr, 10, 64)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
if size == -1 {
|
||||
writeErrorResponse(w, ErrMissingContentLength, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
/// maximum Upload size for objects in a single operation
|
||||
if isMaxObjectSize(size) {
|
||||
writeErrorResponse(w, ErrEntityTooLarge, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract metadata to be saved from incoming HTTP header.
|
||||
metadata := extractMetadataFromHeader(r.Header)
|
||||
if reqAuthType == authTypeStreamingSigned {
|
||||
if contentEncoding, ok := metadata["content-encoding"]; ok {
|
||||
contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
|
||||
if contentEncoding != "" {
|
||||
// Make sure to trim and save the content-encoding
|
||||
// parameter for a streaming signature which is set
|
||||
// to a custom value for example: "aws-chunked,gzip".
|
||||
metadata["content-encoding"] = contentEncoding
|
||||
} else {
|
||||
// Trimmed content encoding is empty when the header
|
||||
// value is set to "aws-chunked" only.
|
||||
|
||||
// Make sure to delete the content-encoding parameter
|
||||
// for a streaming signature which is set to value
|
||||
// for example: "aws-chunked"
|
||||
delete(metadata, "content-encoding")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Make sure we hex encode md5sum here.
|
||||
metadata["etag"] = hex.EncodeToString(md5Bytes)
|
||||
|
||||
sha256sum := ""
|
||||
|
||||
// Lock the object.
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, object)
|
||||
objectLock.Lock()
|
||||
defer objectLock.Unlock()
|
||||
|
||||
var objInfo ObjectInfo
|
||||
switch reqAuthType {
|
||||
case authTypeAnonymous:
|
||||
// Create anonymous object.
|
||||
objInfo, err = objectAPI.AnonPutObject(bucket, object, size, r.Body, metadata, sha256sum)
|
||||
case authTypeStreamingSigned:
|
||||
// Initialize stream signature verifier.
|
||||
reader, s3Error := newSignV4ChunkedReader(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
objInfo, err = objectAPI.PutObject(bucket, object, size, reader, metadata, sha256sum)
|
||||
case authTypeSignedV2, authTypePresignedV2:
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum)
|
||||
case authTypePresigned, authTypeSigned:
|
||||
if s3Error := reqSignatureV4Verify(r, serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
if !skipContentSha256Cksum(r) {
|
||||
sha256sum = r.Header.Get("X-Amz-Content-Sha256")
|
||||
}
|
||||
// Create object.
|
||||
objInfo, err = objectAPI.PutObject(bucket, object, size, r.Body, metadata, sha256sum)
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("ETag", "\""+objInfo.ETag+"\"")
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// HeadObjectHandler - HEAD Object
|
||||
// -----------
|
||||
// The HEAD operation retrieves metadata from an object without returning the object itself.
|
||||
@@ -174,17 +313,23 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeAnonymous:
|
||||
// No verification needed for anonymous requests.
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
@@ -354,37 +499,13 @@ func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
{
|
||||
// FIXME: consolidate bucketPolicy and policy.BucketAccessPolicy so that
|
||||
// the verification below is done on the same type.
|
||||
// Parse bucket policy.
|
||||
policyInfo := &bucketPolicy{}
|
||||
err = parseBucketPolicy(bytes.NewReader(policyBytes), policyInfo)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to parse bucket policy.")
|
||||
writeErrorResponse(w, ErrInvalidPolicyDocument, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Parse check bucket policy.
|
||||
if s3Error := checkBucketPolicyResources(bucket, policyInfo); s3Error != ErrNone {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
policyInfo := &policy.BucketAccessPolicy{}
|
||||
if err = json.Unmarshal(policyBytes, policyInfo); err != nil {
|
||||
policyInfo := policy.BucketAccessPolicy{}
|
||||
if err = json.Unmarshal(policyBytes, &policyInfo); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
var policies []BucketAccessPolicy
|
||||
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
|
||||
policies = append(policies, BucketAccessPolicy{
|
||||
Prefix: prefix,
|
||||
Policy: policy,
|
||||
})
|
||||
}
|
||||
if err = objAPI.SetBucketPolicies(bucket, policies); err != nil {
|
||||
|
||||
if err = objAPI.SetBucketPolicies(bucket, policyInfo); err != nil {
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -453,17 +574,14 @@ func (api gatewayAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
policies, err := objAPI.GetBucketPolicies(bucket)
|
||||
bp, err := objAPI.GetBucketPolicies(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
policyInfo := policy.BucketAccessPolicy{Version: "2012-10-17"}
|
||||
for _, p := range policies {
|
||||
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, p.Policy, bucket, p.Prefix)
|
||||
}
|
||||
policyBytes, err := json.Marshal(&policyInfo)
|
||||
|
||||
policyBytes, err := json.Marshal(bp)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
@@ -499,6 +617,65 @@ func (api gatewayAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWri
|
||||
writeErrorResponse(w, ErrNotImplemented, r.URL)
|
||||
}
|
||||
|
||||
// PutBucketHandler - PUT Bucket
|
||||
// ----------
|
||||
// This implementation of the PUT operation creates a new bucket for authenticated request
|
||||
func (api gatewayAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objectAPI := api.ObjectAPI()
|
||||
if objectAPI == nil {
|
||||
writeErrorResponse(w, ErrServerNotInitialized, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// PutBucket does not have any bucket action.
|
||||
s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
|
||||
if s3Error == ErrInvalidRegion {
|
||||
// Clients like boto3 send putBucket() call signed with region that is configured.
|
||||
s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
|
||||
}
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
vars := router.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// Validate if incoming location constraint is valid, reject
|
||||
// requests which do not follow valid region requirements.
|
||||
location, s3Error := parseLocationConstraint(r)
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// validating region here, because isValidLocationConstraint
|
||||
// reads body which has been read already. So only validating
|
||||
// region here.
|
||||
serverRegion := serverConfig.GetRegion()
|
||||
if serverRegion != location {
|
||||
writeErrorResponse(w, ErrInvalidRegion, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
bucketLock := globalNSMutex.NewNSLock(bucket, "")
|
||||
bucketLock.Lock()
|
||||
defer bucketLock.Unlock()
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucketWithLocation(bucket, location)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to create a bucket.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Make sure to add Location information here only for bucket
|
||||
w.Header().Set("Location", getLocation(r))
|
||||
|
||||
writeSuccessResponseHeadersOnly(w)
|
||||
}
|
||||
|
||||
// DeleteBucketHandler - Delete bucket
|
||||
func (api gatewayAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objectAPI := api.ObjectAPI()
|
||||
@@ -550,17 +727,23 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeAnonymous:
|
||||
// No verification needed for anonymous requests.
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Extract all the litsObjectsV1 query params to their native values.
|
||||
@@ -614,17 +797,23 @@ func (api gatewayAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.R
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeAnonymous:
|
||||
// No verification needed for anonymous requests.
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
@@ -660,7 +849,7 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
|
||||
// Signature V2 validation.
|
||||
s3Error := isReqAuthenticatedV2(r)
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -671,10 +860,16 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
|
||||
s3Error = isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
}
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
case authTypeAnonymous:
|
||||
// No verification needed for anonymous requests.
|
||||
default:
|
||||
// For all unknown auth types return error.
|
||||
writeErrorResponse(w, ErrAccessDenied, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
getBucketInfo := objectAPI.GetBucketInfo
|
||||
|
||||
+194
-65
@@ -17,45 +17,53 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/mc/pkg/console"
|
||||
)
|
||||
|
||||
var gatewayTemplate = `NAME:
|
||||
const azureGatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} BACKEND [ENDPOINT]
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
BACKEND:
|
||||
azure: Microsoft Azure Blob Storage. Default ENDPOINT is https://core.windows.net
|
||||
ENDPOINT:
|
||||
Azure server endpoint. Default ENDPOINT is https://core.windows.net
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of your storage backend.
|
||||
MINIO_SECRET_KEY: Password or secret key of your storage backend.
|
||||
MINIO_ACCESS_KEY: Username or access key of Azure storage.
|
||||
MINIO_SECRET_KEY: Password or secret key of Azure storage.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}} azure
|
||||
$ {{.HelpName}}
|
||||
2. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
$ export MINIO_SECRET_KEY=azureaccountkey
|
||||
$ {{.HelpName}} https://azure.example.com
|
||||
`
|
||||
|
||||
var gatewayCmd = cli.Command{
|
||||
Name: "gateway",
|
||||
Usage: "Start object storage gateway.",
|
||||
Action: gatewayMain,
|
||||
CustomHelpTemplate: gatewayTemplate,
|
||||
var azureBackendCmd = cli.Command{
|
||||
Name: "azure",
|
||||
Usage: "Microsoft Azure Blob Storage.",
|
||||
Action: azureGatewayMain,
|
||||
CustomHelpTemplate: azureGatewayTemplate,
|
||||
Flags: append(serverFlags,
|
||||
cli.BoolFlag{
|
||||
Name: "quiet",
|
||||
@@ -65,11 +73,65 @@ var gatewayCmd = cli.Command{
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
|
||||
const s3GatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENDPOINT:
|
||||
S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of S3 storage.
|
||||
MINIO_SECRET_KEY: Password or secret key of S3 storage.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for S3 backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ {{.HelpName}} https://play.minio.io:9000
|
||||
`
|
||||
|
||||
var s3BackendCmd = cli.Command{
|
||||
Name: "s3",
|
||||
Usage: "Amazon Simple Storage Service (S3).",
|
||||
Action: s3GatewayMain,
|
||||
CustomHelpTemplate: s3GatewayTemplate,
|
||||
Flags: append(serverFlags,
|
||||
cli.BoolFlag{
|
||||
Name: "quiet",
|
||||
Usage: "Disable startup banner.",
|
||||
},
|
||||
),
|
||||
HideHelpCommand: true,
|
||||
}
|
||||
|
||||
var gatewayCmd = cli.Command{
|
||||
Name: "gateway",
|
||||
Usage: "Start object storage gateway.",
|
||||
HideHelpCommand: true,
|
||||
Subcommands: []cli.Command{azureBackendCmd, s3BackendCmd},
|
||||
}
|
||||
|
||||
// Represents the type of the gateway backend.
|
||||
type gatewayBackend string
|
||||
|
||||
const (
|
||||
azureBackend gatewayBackend = "azure"
|
||||
s3Backend gatewayBackend = "s3"
|
||||
// Add more backends here.
|
||||
)
|
||||
|
||||
@@ -79,21 +141,41 @@ func mustGetGatewayCredsFromEnv() (accessKey, secretKey string) {
|
||||
accessKey = os.Getenv("MINIO_ACCESS_KEY")
|
||||
secretKey = os.Getenv("MINIO_SECRET_KEY")
|
||||
if accessKey == "" || secretKey == "" {
|
||||
console.Fatalln("Access and secret keys are mandatory to run Minio gateway server.")
|
||||
fatalIf(errors.New("Missing credentials"), "Access and secret keys are mandatory to run Minio gateway server.")
|
||||
}
|
||||
return accessKey, secretKey
|
||||
}
|
||||
|
||||
// Set browser setting from environment variables
|
||||
func mustSetBrowserSettingFromEnv() {
|
||||
if browser := os.Getenv("MINIO_BROWSER"); browser != "" {
|
||||
browserFlag, err := ParseBrowserFlag(browser)
|
||||
if err != nil {
|
||||
fatalIf(errors.New("invalid value"), "Unknown value ‘%s’ in MINIO_BROWSER environment variable.", browser)
|
||||
}
|
||||
|
||||
// browser Envs are set globally, this does not represent
|
||||
// if browser is turned off or on.
|
||||
globalIsEnvBrowser = true
|
||||
globalIsBrowserEnabled = bool(browserFlag)
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize gateway layer depending on the backend type.
|
||||
// Supported backend types are
|
||||
//
|
||||
// - Azure Blob Storage.
|
||||
// - Add your favorite backend here.
|
||||
func newGatewayLayer(backendType, endPoint, accessKey, secretKey string, secure bool) (GatewayLayer, error) {
|
||||
if gatewayBackend(backendType) != azureBackend {
|
||||
return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
|
||||
func newGatewayLayer(backendType gatewayBackend, endpoint, accessKey, secretKey string, secure bool) (GatewayLayer, error) {
|
||||
|
||||
switch gatewayBackend(backendType) {
|
||||
case azureBackend:
|
||||
return newAzureLayer(endpoint, accessKey, secretKey, secure)
|
||||
case s3Backend:
|
||||
return newS3Gateway(endpoint, accessKey, secretKey, secure)
|
||||
}
|
||||
return newAzureLayer(endPoint, accessKey, secretKey, secure)
|
||||
|
||||
return nil, fmt.Errorf("Unrecognized backend type %s", backendType)
|
||||
}
|
||||
|
||||
// Initialize a new gateway config.
|
||||
@@ -102,7 +184,7 @@ func newGatewayLayer(backendType, endPoint, accessKey, secretKey string, secure
|
||||
// only used in memory.
|
||||
func newGatewayConfig(accessKey, secretKey, region string) error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfigV14()
|
||||
srvCfg := newServerConfigV18()
|
||||
|
||||
// If env is set for a fresh start, save them to config file.
|
||||
srvCfg.SetCredential(credential{
|
||||
@@ -110,17 +192,9 @@ func newGatewayConfig(accessKey, secretKey, region string) error {
|
||||
SecretKey: secretKey,
|
||||
})
|
||||
|
||||
// Set default printing to console.
|
||||
srvCfg.Logger.SetConsole(consoleLogger{true, "error"})
|
||||
|
||||
// Set custom region.
|
||||
srvCfg.SetRegion(region)
|
||||
|
||||
// Create certs path for SSL configuration.
|
||||
if err := createConfigDir(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
// Save the new config globally.
|
||||
// unlock the mutex.
|
||||
@@ -138,6 +212,7 @@ func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error)
|
||||
// Default connection will be "secure".
|
||||
arg = "https://" + arg
|
||||
}
|
||||
|
||||
u, err := url.Parse(arg)
|
||||
if err != nil {
|
||||
return "", false, err
|
||||
@@ -153,55 +228,100 @@ func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error)
|
||||
}
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway'.
|
||||
func gatewayMain(ctx *cli.Context) {
|
||||
if !ctx.Args().Present() || ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, "gateway", 1)
|
||||
// Validate gateway arguments.
|
||||
func validateGatewayArguments(serverAddr, endpointAddr string) error {
|
||||
if err := CheckLocalServerAddr(serverAddr); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if runtime.GOOS == "darwin" {
|
||||
_, port := mustSplitHostPort(serverAddr)
|
||||
// On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back
|
||||
// to IPv6 address i.e minio will start listening on IPv6 address whereas another
|
||||
// (non-)minio process is listening on IPv4 of given port.
|
||||
// To avoid this error situation we check for port availability only for macOS.
|
||||
if err := checkPortAvailability(port); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if endpointAddr != "" {
|
||||
// Reject the endpoint if it points to the gateway handler itself.
|
||||
sameTarget, err := sameLocalAddrs(endpointAddr, serverAddr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if sameTarget {
|
||||
return errors.New("endpoint points to the local gateway")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway azure' command line.
|
||||
func azureGatewayMain(ctx *cli.Context) {
|
||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, "azure", 1)
|
||||
}
|
||||
|
||||
gatewayMain(ctx, azureBackend)
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway s3' command line.
|
||||
func s3GatewayMain(ctx *cli.Context) {
|
||||
if ctx.Args().Present() && ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, "s3", 1)
|
||||
}
|
||||
|
||||
gatewayMain(ctx, s3Backend)
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway'.
|
||||
func gatewayMain(ctx *cli.Context, backendType gatewayBackend) {
|
||||
// Fetch access and secret key from env.
|
||||
accessKey, secretKey := mustGetGatewayCredsFromEnv()
|
||||
|
||||
// Initialize new gateway config.
|
||||
//
|
||||
// TODO: add support for custom region when we add
|
||||
// support for S3 backend storage, currently this can
|
||||
// default to "us-east-1"
|
||||
err := newGatewayConfig(accessKey, secretKey, "us-east-1")
|
||||
if err != nil {
|
||||
console.Fatalf("Unable to initialize gateway config. Error: %s", err)
|
||||
}
|
||||
// Fetch browser env setting
|
||||
mustSetBrowserSettingFromEnv()
|
||||
|
||||
// Enable console logging.
|
||||
enableConsoleLogger()
|
||||
// Initialize new gateway config.
|
||||
|
||||
newGatewayConfig(accessKey, secretKey, globalMinioDefaultRegion)
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
|
||||
if quietFlag {
|
||||
log.EnableQuiet()
|
||||
}
|
||||
|
||||
// First argument is selected backend type.
|
||||
backendType := ctx.Args().First()
|
||||
serverAddr := ctx.String("address")
|
||||
endpointAddr := ctx.Args().Get(0)
|
||||
err := validateGatewayArguments(serverAddr, endpointAddr)
|
||||
fatalIf(err, "Invalid argument")
|
||||
|
||||
// Second argument is endpoint. If no endpoint is specified then the
|
||||
// gateway implementation should use a default setting.
|
||||
endPoint, secure, err := parseGatewayEndpoint(ctx.Args().Get(1))
|
||||
if err != nil {
|
||||
console.Fatalf("Unable to parse endpoint. Error: %s", err)
|
||||
}
|
||||
endPoint, secure, err := parseGatewayEndpoint(endpointAddr)
|
||||
fatalIf(err, "Unable to parse endpoint")
|
||||
|
||||
// Create certs path for SSL configuration.
|
||||
err = createConfigDir()
|
||||
if err != nil {
|
||||
console.Fatalf("Unable to create configuration directory. Error: %s", err)
|
||||
}
|
||||
fatalIf(createConfigDir(), "Unable to create configuration directory")
|
||||
|
||||
newObject, err := newGatewayLayer(backendType, endPoint, accessKey, secretKey, secure)
|
||||
if err != nil {
|
||||
console.Fatalf("Unable to initialize gateway layer. Error: %s", err)
|
||||
}
|
||||
fatalIf(err, "Unable to initialize gateway layer")
|
||||
|
||||
initNSLock(false) // Enable local namespace lock.
|
||||
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
|
||||
// credentials Envs are set globally.
|
||||
globalIsEnvCreds = true
|
||||
|
||||
// Register web router when its enabled.
|
||||
if globalIsBrowserEnabled {
|
||||
aerr := registerWebRouter(router)
|
||||
fatalIf(aerr, "Unable to configure web browser")
|
||||
}
|
||||
registerGatewayAPIRouter(router, newObject)
|
||||
|
||||
var handlerFns = []HandlerFunc{
|
||||
@@ -212,6 +332,13 @@ func gatewayMain(ctx *cli.Context) {
|
||||
// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
|
||||
setCrossDomainPolicy,
|
||||
// Validates all incoming requests to have a valid date header.
|
||||
// Redirect some pre-defined browser request paths to a static location prefix.
|
||||
setBrowserRedirectHandler,
|
||||
// Validates if incoming request is for restricted buckets.
|
||||
setPrivateBucketHandler,
|
||||
// Adds cache control for all browser requests.
|
||||
setBrowserCacheControlHandler,
|
||||
// Validates all incoming requests to have a valid date header.
|
||||
setTimeValidityHandler,
|
||||
// CORS setting for all browser API requests.
|
||||
setCorsHandler,
|
||||
@@ -222,12 +349,14 @@ func gatewayMain(ctx *cli.Context) {
|
||||
// routes them accordingly. Client receives a HTTP error for
|
||||
// invalid/unsupported signatures.
|
||||
setAuthHandler,
|
||||
// Add new handlers here.
|
||||
|
||||
}
|
||||
|
||||
apiServer := NewServerMux(ctx.String("address"), registerHandlers(router, handlerFns...))
|
||||
apiServer := NewServerMux(serverAddr, registerHandlers(router, handlerFns...))
|
||||
|
||||
// Set if we are SSL enabled S3 gateway.
|
||||
globalIsSSL = isSSL()
|
||||
_, _, globalIsSSL, err = getSSLConfig()
|
||||
fatalIf(err, "Invalid SSL key file")
|
||||
|
||||
// Start server, automatically configures TLS if certs are available.
|
||||
go func() {
|
||||
@@ -235,13 +364,10 @@ func gatewayMain(ctx *cli.Context) {
|
||||
if globalIsSSL {
|
||||
cert, key = getPublicCertFile(), getPrivateKeyFile()
|
||||
}
|
||||
if aerr := apiServer.ListenAndServe(cert, key); aerr != nil {
|
||||
console.Fatalf("Failed to start minio server. Error: %s\n", aerr)
|
||||
}
|
||||
}()
|
||||
|
||||
apiEndPoints, err := finalizeAPIEndpoints(apiServer.Addr)
|
||||
fatalIf(err, "Unable to finalize API endpoints for %s", apiServer.Addr)
|
||||
aerr := apiServer.ListenAndServe(cert, key)
|
||||
fatalIf(aerr, "Failed to start minio server")
|
||||
}()
|
||||
|
||||
// Once endpoints are finalized, initialize the new object api.
|
||||
globalObjLayerMutex.Lock()
|
||||
@@ -253,9 +379,12 @@ func gatewayMain(ctx *cli.Context) {
|
||||
mode := ""
|
||||
if gatewayBackend(backendType) == azureBackend {
|
||||
mode = globalMinioModeGatewayAzure
|
||||
} else if gatewayBackend(backendType) == s3Backend {
|
||||
mode = globalMinioModeGatewayS3
|
||||
}
|
||||
checkUpdate(mode)
|
||||
printGatewayStartupMessage(apiEndPoints, accessKey, secretKey, backendType)
|
||||
apiEndpoints := getAPIEndpoints(apiServer.Addr)
|
||||
printGatewayStartupMessage(apiEndpoints, accessKey, secretKey, backendType)
|
||||
}
|
||||
|
||||
<-globalServiceDoneCh
|
||||
|
||||
@@ -16,7 +16,11 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Test parseGatewayEndpoint
|
||||
func TestParseGatewayEndpoint(t *testing.T) {
|
||||
@@ -48,3 +52,57 @@ func TestParseGatewayEndpoint(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestSetBrowserFromEnv(t *testing.T) {
|
||||
browser := os.Getenv("MINIO_BROWSER")
|
||||
|
||||
os.Setenv("MINIO_BROWSER", "on")
|
||||
mustSetBrowserSettingFromEnv()
|
||||
if globalIsBrowserEnabled != true {
|
||||
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, false)
|
||||
}
|
||||
|
||||
os.Setenv("MINIO_BROWSER", "off")
|
||||
mustSetBrowserSettingFromEnv()
|
||||
if globalIsBrowserEnabled != false {
|
||||
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true)
|
||||
}
|
||||
os.Setenv("MINIO_BROWSER", "")
|
||||
mustSetBrowserSettingFromEnv()
|
||||
if globalIsBrowserEnabled != false {
|
||||
t.Errorf("Expected the response status to be `%t`, but instead found `%t`", globalIsBrowserEnabled, true)
|
||||
}
|
||||
os.Setenv("MINIO_BROWSER", browser)
|
||||
}
|
||||
|
||||
// Test validateGatewayArguments
|
||||
func TestValidateGatewayArguments(t *testing.T) {
|
||||
nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool {
|
||||
return !strings.HasPrefix(ip, "127.")
|
||||
}, "")
|
||||
if len(nonLoopBackIPs) == 0 {
|
||||
t.Fatalf("No non-loop back IP address found for this host")
|
||||
}
|
||||
nonLoopBackIP := nonLoopBackIPs.ToSlice()[0]
|
||||
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
endpointAddr string
|
||||
valid bool
|
||||
}{
|
||||
{":9000", "http://localhost:9001", true},
|
||||
{":9000", "http://google.com", true},
|
||||
{"123.123.123.123:9000", "http://localhost:9000", false},
|
||||
{":9000", "http://localhost:9000", false},
|
||||
{":9000", nonLoopBackIP + ":9000", false},
|
||||
}
|
||||
for i, test := range testCases {
|
||||
err := validateGatewayArguments(test.serverAddr, test.endpointAddr)
|
||||
if test.valid && err != nil {
|
||||
t.Errorf("Test %d expected not to return error but got %s", i+1, err)
|
||||
}
|
||||
if !test.valid && err == nil {
|
||||
t.Errorf("Test %d expected to fail but it did not", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,15 +20,20 @@ import (
|
||||
"io"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
)
|
||||
|
||||
// GatewayLayer - Interface to implement gateway mode.
|
||||
type GatewayLayer interface {
|
||||
ObjectLayer
|
||||
|
||||
AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
|
||||
AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
|
||||
SetBucketPolicies(string, []BucketAccessPolicy) error
|
||||
GetBucketPolicies(string) ([]BucketAccessPolicy, error)
|
||||
|
||||
AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error)
|
||||
|
||||
SetBucketPolicies(string, policy.BucketAccessPolicy) error
|
||||
GetBucketPolicies(string) (policy.BucketAccessPolicy, error)
|
||||
DeleteBucketPolicies(string) error
|
||||
AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
|
||||
AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error)
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"io"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
// AnonPutObject creates a new object anonymously with the incoming data,
|
||||
func (l *s3Objects) AnonPutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) {
|
||||
var sha256sumBytes []byte
|
||||
|
||||
var err error
|
||||
if sha256sum != "" {
|
||||
sha256sumBytes, err = hex.DecodeString(sha256sum)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
var md5sumBytes []byte
|
||||
md5sum := metadata["etag"]
|
||||
if md5sum != "" {
|
||||
md5sumBytes, err = hex.DecodeString(md5sum)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
delete(metadata, "etag")
|
||||
}
|
||||
|
||||
oi, err := l.anonClient.PutObject(bucket, object, size, data, md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata))
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
|
||||
// AnonGetObject - Get object anonymously
|
||||
func (l *s3Objects) AnonGetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
|
||||
r := minio.NewGetReqHeaders()
|
||||
if err := r.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
object, _, err := l.anonClient.GetObject(bucket, key, r)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
|
||||
defer object.Close()
|
||||
|
||||
if _, err := io.CopyN(writer, object, length); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// AnonGetObjectInfo - Get object info anonymously
|
||||
func (l *s3Objects) AnonGetObjectInfo(bucket string, object string) (ObjectInfo, error) {
|
||||
r := minio.NewHeadReqHeaders()
|
||||
oi, err := l.anonClient.StatObject(bucket, object, r)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
|
||||
// AnonListObjects - List objects anonymously
|
||||
func (l *s3Objects) AnonListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
result, err := l.anonClient.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
return ListObjectsInfo{}, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
return fromMinioClientListBucketResult(bucket, result), nil
|
||||
}
|
||||
|
||||
// AnonGetBucketInfo - Get bucket metadata anonymously.
|
||||
func (l *s3Objects) AnonGetBucketInfo(bucket string) (BucketInfo, error) {
|
||||
if exists, err := l.anonClient.BucketExists(bucket); err != nil {
|
||||
return BucketInfo{}, s3ToObjectError(traceError(err), bucket)
|
||||
} else if !exists {
|
||||
return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
|
||||
}
|
||||
|
||||
buckets, err := l.anonClient.ListBuckets()
|
||||
if err != nil {
|
||||
return BucketInfo{}, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
for _, bi := range buckets {
|
||||
if bi.Name != bucket {
|
||||
continue
|
||||
}
|
||||
|
||||
return BucketInfo{
|
||||
Name: bi.Name,
|
||||
Created: bi.CreationDate,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (l *s3Objects) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (l *s3Objects) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return []BucketInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (l *s3Objects) HealObject(bucket string, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (l *s3Objects) ListObjectsHeal(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
return ListObjectsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (l *s3Objects) ListUploadsHeal(bucket string, prefix string, marker string, uploadIDMarker string, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
return ListMultipartsInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
@@ -0,0 +1,568 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
)
|
||||
|
||||
// s3ToObjectError converts Minio errors to minio object layer errors.
|
||||
func s3ToObjectError(err error, params ...string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
e, ok := err.(*Error)
|
||||
if !ok {
|
||||
// Code should be fixed if this function is called without doing traceError()
|
||||
// Else handling different situations in this function makes this function complicated.
|
||||
errorIf(err, "Expected type *Error")
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.e
|
||||
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
|
||||
minioErr, ok := err.(minio.ErrorResponse)
|
||||
if !ok {
|
||||
// We don't interpret non Minio errors. As minio errors will
|
||||
// have StatusCode to help to convert to object errors.
|
||||
return e
|
||||
}
|
||||
|
||||
switch minioErr.Code {
|
||||
case "BucketAlreadyOwnedByYou":
|
||||
err = BucketAlreadyOwnedByYou{}
|
||||
case "BucketNotEmpty":
|
||||
err = BucketNotEmpty{}
|
||||
case "NoSuchBucketPolicy":
|
||||
err = PolicyNotFound{}
|
||||
case "InvalidBucketName":
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
case "NoSuchBucket":
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
case "NoSuchKey":
|
||||
if object != "" {
|
||||
err = ObjectNotFound{Bucket: bucket, Object: object}
|
||||
} else {
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
}
|
||||
case "XMinioInvalidObjectName":
|
||||
err = ObjectNameInvalid{}
|
||||
case "AccessDenied":
|
||||
err = PrefixAccessDenied{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
case "XAmzContentSHA256Mismatch":
|
||||
err = SHA256Mismatch{}
|
||||
}
|
||||
|
||||
e.e = err
|
||||
return e
|
||||
}
|
||||
|
||||
// s3Objects implements gateway for Minio and S3 compatible object storage servers.
|
||||
type s3Objects struct {
|
||||
Client *minio.Core
|
||||
anonClient *minio.Core
|
||||
}
|
||||
|
||||
// newS3Gateway returns s3 gatewaylayer
|
||||
func newS3Gateway(endpoint string, accessKey, secretKey string, secure bool) (GatewayLayer, error) {
|
||||
if endpoint == "" {
|
||||
endpoint = "s3.amazonaws.com"
|
||||
secure = true
|
||||
}
|
||||
|
||||
// Initialize minio client object.
|
||||
client, err := minio.NewCore(endpoint, accessKey, secretKey, secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
anonClient, err := minio.NewCore(endpoint, "", "", secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &s3Objects{
|
||||
Client: client,
|
||||
anonClient: anonClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Shutdown saves any gateway metadata to disk
|
||||
// if necessary and reload upon next restart.
|
||||
func (l *s3Objects) Shutdown() error {
|
||||
// TODO
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageInfo is not relevant to S3 backend.
|
||||
func (l *s3Objects) StorageInfo() StorageInfo {
|
||||
return StorageInfo{}
|
||||
}
|
||||
|
||||
// MakeBucket creates a new container on S3 backend.
|
||||
func (l *s3Objects) MakeBucketWithLocation(bucket, location string) error {
|
||||
err := l.Client.MakeBucket(bucket, location)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// GetBucketInfo gets bucket metadata..
|
||||
func (l *s3Objects) GetBucketInfo(bucket string) (BucketInfo, error) {
|
||||
buckets, err := l.Client.ListBuckets()
|
||||
if err != nil {
|
||||
return BucketInfo{}, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
for _, bi := range buckets {
|
||||
if bi.Name != bucket {
|
||||
continue
|
||||
}
|
||||
|
||||
return BucketInfo{
|
||||
Name: bi.Name,
|
||||
Created: bi.CreationDate,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return BucketInfo{}, traceError(BucketNotFound{Bucket: bucket})
|
||||
}
|
||||
|
||||
// ListBuckets lists all S3 buckets
|
||||
func (l *s3Objects) ListBuckets() ([]BucketInfo, error) {
|
||||
buckets, err := l.Client.ListBuckets()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
b := make([]BucketInfo, len(buckets))
|
||||
for i, bi := range buckets {
|
||||
b[i] = BucketInfo{
|
||||
Name: bi.Name,
|
||||
Created: bi.CreationDate,
|
||||
}
|
||||
}
|
||||
|
||||
return b, err
|
||||
}
|
||||
|
||||
// DeleteBucket deletes a bucket on S3
|
||||
func (l *s3Objects) DeleteBucket(bucket string) error {
|
||||
err := l.Client.RemoveBucket(bucket)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListObjects lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
result, err := l.Client.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
return ListObjectsInfo{}, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
return fromMinioClientListBucketResult(bucket, result), nil
|
||||
}
|
||||
|
||||
// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken string, fetchOwner bool, delimiter string, maxKeys int) (ListObjectsV2Info, error) {
|
||||
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
return ListObjectsV2Info{}, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
return fromMinioClientListBucketV2Result(bucket, result), nil
|
||||
}
|
||||
|
||||
// fromMinioClientListBucketV2Result converts minio ListBucketResult to ListObjectsInfo
|
||||
func fromMinioClientListBucketV2Result(bucket string, result minio.ListBucketV2Result) ListObjectsV2Info {
|
||||
objects := make([]ObjectInfo, len(result.Contents))
|
||||
|
||||
for i, oi := range result.Contents {
|
||||
objects[i] = fromMinioClientObjectInfo(bucket, oi)
|
||||
}
|
||||
|
||||
prefixes := make([]string, len(result.CommonPrefixes))
|
||||
for i, p := range result.CommonPrefixes {
|
||||
prefixes[i] = p.Prefix
|
||||
}
|
||||
|
||||
return ListObjectsV2Info{
|
||||
IsTruncated: result.IsTruncated,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
|
||||
ContinuationToken: result.ContinuationToken,
|
||||
NextContinuationToken: result.NextContinuationToken,
|
||||
}
|
||||
}
|
||||
|
||||
// fromMinioClientListBucketResult converts minio ListBucketResult to ListObjectsInfo
|
||||
func fromMinioClientListBucketResult(bucket string, result minio.ListBucketResult) ListObjectsInfo {
|
||||
objects := make([]ObjectInfo, len(result.Contents))
|
||||
|
||||
for i, oi := range result.Contents {
|
||||
objects[i] = fromMinioClientObjectInfo(bucket, oi)
|
||||
}
|
||||
|
||||
prefixes := make([]string, len(result.CommonPrefixes))
|
||||
for i, p := range result.CommonPrefixes {
|
||||
prefixes[i] = p.Prefix
|
||||
}
|
||||
|
||||
return ListObjectsInfo{
|
||||
IsTruncated: result.IsTruncated,
|
||||
NextMarker: result.NextMarker,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
}
|
||||
}
|
||||
|
||||
// GetObject reads an object from S3. Supports additional
|
||||
// parameters like offset and length which are synonymous with
|
||||
// HTTP Range requests.
|
||||
//
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
|
||||
r := minio.NewGetReqHeaders()
|
||||
|
||||
if length < 0 && length != -1 {
|
||||
return s3ToObjectError(traceError(errInvalidArgument), bucket, key)
|
||||
}
|
||||
|
||||
if startOffset >= 0 && length >= 0 {
|
||||
if err := r.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
}
|
||||
object, _, err := l.Client.GetObject(bucket, key, r)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
|
||||
defer object.Close()
|
||||
|
||||
if _, err := io.Copy(writer, object); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
|
||||
func fromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
|
||||
userDefined := fromMinioClientMetadata(oi.Metadata)
|
||||
userDefined["Content-Type"] = oi.ContentType
|
||||
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: oi.Key,
|
||||
ModTime: oi.LastModified,
|
||||
Size: oi.Size,
|
||||
ETag: oi.ETag,
|
||||
UserDefined: userDefined,
|
||||
ContentType: oi.ContentType,
|
||||
ContentEncoding: oi.Metadata.Get("Content-Encoding"),
|
||||
}
|
||||
}
|
||||
|
||||
// GetObjectInfo reads object info and replies back ObjectInfo
|
||||
func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
|
||||
r := minio.NewHeadReqHeaders()
|
||||
oi, err := l.Client.StatObject(bucket, object, r)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data,
|
||||
func (l *s3Objects) PutObject(bucket string, object string, size int64, data io.Reader, metadata map[string]string, sha256sum string) (ObjectInfo, error) {
|
||||
var sha256sumBytes []byte
|
||||
|
||||
var err error
|
||||
if sha256sum != "" {
|
||||
sha256sumBytes, err = hex.DecodeString(sha256sum)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
var md5sumBytes []byte
|
||||
md5sum := metadata["etag"]
|
||||
if md5sum != "" {
|
||||
md5sumBytes, err = hex.DecodeString(md5sum)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
delete(metadata, "etag")
|
||||
}
|
||||
|
||||
oi, err := l.Client.PutObject(bucket, object, size, data, md5sumBytes, sha256sumBytes, toMinioClientMetadata(metadata))
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
|
||||
// CopyObject copies a blob from source container to destination container.
|
||||
func (l *s3Objects) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string, metadata map[string]string) (ObjectInfo, error) {
|
||||
err := l.Client.CopyObject(destBucket, destObject, path.Join(srcBucket, srcObject), minio.CopyConditions{})
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), srcBucket, srcObject)
|
||||
}
|
||||
|
||||
oi, err := l.GetObjectInfo(destBucket, destObject)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), destBucket, destObject)
|
||||
}
|
||||
|
||||
return oi, nil
|
||||
}
|
||||
|
||||
// DeleteObject deletes a blob in bucket
|
||||
func (l *s3Objects) DeleteObject(bucket string, object string) error {
|
||||
err := l.Client.RemoveObject(bucket, object)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fromMinioClientUploadMetadata converts ObjectMultipartInfo to uploadMetadata
|
||||
func fromMinioClientUploadMetadata(omi minio.ObjectMultipartInfo) uploadMetadata {
|
||||
return uploadMetadata{
|
||||
Object: omi.Key,
|
||||
UploadID: omi.UploadID,
|
||||
Initiated: omi.Initiated,
|
||||
}
|
||||
}
|
||||
|
||||
// fromMinioClientListMultipartsInfo converts minio ListMultipartUploadsResult to ListMultipartsInfo
|
||||
func fromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) ListMultipartsInfo {
|
||||
uploads := make([]uploadMetadata, len(lmur.Uploads))
|
||||
|
||||
for i, um := range lmur.Uploads {
|
||||
uploads[i] = fromMinioClientUploadMetadata(um)
|
||||
}
|
||||
|
||||
commonPrefixes := make([]string, len(lmur.CommonPrefixes))
|
||||
for i, cp := range lmur.CommonPrefixes {
|
||||
commonPrefixes[i] = cp.Prefix
|
||||
}
|
||||
|
||||
return ListMultipartsInfo{
|
||||
KeyMarker: lmur.KeyMarker,
|
||||
UploadIDMarker: lmur.UploadIDMarker,
|
||||
NextKeyMarker: lmur.NextKeyMarker,
|
||||
NextUploadIDMarker: lmur.NextUploadIDMarker,
|
||||
MaxUploads: int(lmur.MaxUploads),
|
||||
IsTruncated: lmur.IsTruncated,
|
||||
Uploads: uploads,
|
||||
Prefix: lmur.Prefix,
|
||||
Delimiter: lmur.Delimiter,
|
||||
CommonPrefixes: commonPrefixes,
|
||||
EncodingType: lmur.EncodingType,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ListMultipartUploads lists all multipart uploads.
|
||||
func (l *s3Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
|
||||
result, err := l.Client.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err != nil {
|
||||
return ListMultipartsInfo{}, err
|
||||
}
|
||||
|
||||
return fromMinioClientListMultipartsInfo(result), nil
|
||||
}
|
||||
|
||||
// fromMinioClientMetadata converts minio metadata to map[string]string
|
||||
func fromMinioClientMetadata(metadata map[string][]string) map[string]string {
|
||||
mm := map[string]string{}
|
||||
for k, v := range metadata {
|
||||
mm[http.CanonicalHeaderKey(k)] = v[0]
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// toMinioClientMetadata converts metadata to map[string][]string
|
||||
func toMinioClientMetadata(metadata map[string]string) map[string][]string {
|
||||
mm := map[string][]string{}
|
||||
for k, v := range metadata {
|
||||
mm[http.CanonicalHeaderKey(k)] = []string{v}
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// NewMultipartUpload upload object in multiple parts
|
||||
func (l *s3Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
return l.Client.NewMultipartUpload(bucket, object, toMinioClientMetadata(metadata))
|
||||
}
|
||||
|
||||
// CopyObjectPart copy part of object to other bucket and object
|
||||
func (l *s3Objects) CopyObjectPart(srcBucket string, srcObject string, destBucket string, destObject string, uploadID string, partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
// FIXME: implement CopyObjectPart
|
||||
return PartInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
// fromMinioClientObjectPart converts minio ObjectPart to PartInfo
|
||||
func fromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
|
||||
return PartInfo{
|
||||
Size: op.Size,
|
||||
ETag: canonicalizeETag(op.ETag),
|
||||
LastModified: op.LastModified,
|
||||
PartNumber: op.PartNumber,
|
||||
}
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, size int64, data io.Reader, md5Hex string, sha256sum string) (PartInfo, error) {
|
||||
md5HexBytes, err := hex.DecodeString(md5Hex)
|
||||
if err != nil {
|
||||
return PartInfo{}, err
|
||||
}
|
||||
|
||||
sha256sumBytes, err := hex.DecodeString(sha256sum)
|
||||
if err != nil {
|
||||
return PartInfo{}, err
|
||||
}
|
||||
|
||||
info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, size, data, md5HexBytes, sha256sumBytes)
|
||||
if err != nil {
|
||||
return PartInfo{}, err
|
||||
}
|
||||
|
||||
return fromMinioClientObjectPart(info), nil
|
||||
}
|
||||
|
||||
// fromMinioClientObjectParts converts minio ObjectPart to PartInfo
|
||||
func fromMinioClientObjectParts(parts []minio.ObjectPart) []PartInfo {
|
||||
toParts := make([]PartInfo, len(parts))
|
||||
for i, part := range parts {
|
||||
toParts[i] = fromMinioClientObjectPart(part)
|
||||
}
|
||||
return toParts
|
||||
}
|
||||
|
||||
// fromMinioClientListPartsInfo converts minio ListObjectPartsResult to ListPartsInfo
|
||||
func fromMinioClientListPartsInfo(lopr minio.ListObjectPartsResult) ListPartsInfo {
|
||||
return ListPartsInfo{
|
||||
UploadID: lopr.UploadID,
|
||||
Bucket: lopr.Bucket,
|
||||
Object: lopr.Key,
|
||||
StorageClass: "",
|
||||
PartNumberMarker: lopr.PartNumberMarker,
|
||||
NextPartNumberMarker: lopr.NextPartNumberMarker,
|
||||
MaxParts: lopr.MaxParts,
|
||||
IsTruncated: lopr.IsTruncated,
|
||||
EncodingType: lopr.EncodingType,
|
||||
Parts: fromMinioClientObjectParts(lopr.ObjectParts),
|
||||
}
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (l *s3Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (ListPartsInfo, error) {
|
||||
result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
return ListPartsInfo{}, err
|
||||
}
|
||||
|
||||
return fromMinioClientListPartsInfo(result), nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload aborts a ongoing multipart upload
|
||||
func (l *s3Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error {
|
||||
return l.Client.AbortMultipartUpload(bucket, object, uploadID)
|
||||
}
|
||||
|
||||
// toMinioClientCompletePart converts completePart to minio CompletePart
|
||||
func toMinioClientCompletePart(part completePart) minio.CompletePart {
|
||||
return minio.CompletePart{
|
||||
ETag: part.ETag,
|
||||
PartNumber: part.PartNumber,
|
||||
}
|
||||
}
|
||||
|
||||
// toMinioClientCompleteParts converts []completePart to minio []CompletePart
|
||||
func toMinioClientCompleteParts(parts []completePart) []minio.CompletePart {
|
||||
mparts := make([]minio.CompletePart, len(parts))
|
||||
for i, part := range parts {
|
||||
mparts[i] = toMinioClientCompletePart(part)
|
||||
}
|
||||
return mparts
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (l *s3Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []completePart) (ObjectInfo, error) {
|
||||
err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, toMinioClientCompleteParts(uploadedParts))
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return l.GetObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
// SetBucketPolicies sets policy on bucket
|
||||
func (l *s3Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
|
||||
if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, "")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBucketPolicies will get policy on bucket
|
||||
func (l *s3Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
|
||||
policyInfo, err := l.Client.GetBucketPolicy(bucket)
|
||||
if err != nil {
|
||||
return policy.BucketAccessPolicy{}, s3ToObjectError(traceError(err), bucket, "")
|
||||
}
|
||||
return policyInfo, nil
|
||||
}
|
||||
|
||||
// DeleteBucketPolicies deletes all policies on bucket
|
||||
func (l *s3Objects) DeleteBucketPolicies(bucket string) error {
|
||||
if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user