mirror of
https://github.com/pgsty/minio.git
synced 2026-11-15 03:55:31 +03:00
Compare commits
125 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 289457568c | |||
| fd3897d0c3 | |||
| 047b7aff0c | |||
| 1164fc60f3 | |||
| 48218272cc | |||
| 0c880bb852 | |||
| 1ebbc2ce88 | |||
| da2faa19a1 | |||
| 2afd196c83 | |||
| b606ba3f81 | |||
| 3316dbc037 | |||
| 033cfb5cef | |||
| 3b2486ebaf | |||
| 018813b98f | |||
| e608e05cda | |||
| 3ea28e9771 | |||
| 45c35b3544 | |||
| 09a9002f12 | |||
| 5cdcc73bd5 | |||
| 2dd117f647 | |||
| 2d19663fef | |||
| 254b05e314 | |||
| a003de72bf | |||
| a337ea4d11 | |||
| f3f09ed14e | |||
| bb73c84b10 | |||
| e19eddd759 | |||
| 7f99cc9768 | |||
| 2760409656 | |||
| 1f13235cbd | |||
| dd5a3289dd | |||
| b6e4f053a3 | |||
| 1c3f55ff64 | |||
| e2d5a87b26 | |||
| d0a43af616 | |||
| a020a70484 | |||
| 3f09c17bfe | |||
| 24d9d7e5fa | |||
| 85580fe0d6 | |||
| 8a1dc10c60 | |||
| 17301fe45d | |||
| aa7e5c71e9 | |||
| 78a641fc6a | |||
| 12f67d47f1 | |||
| 0bb6247056 | |||
| dd202a1a5f | |||
| ede504400f | |||
| 4b2d04c86f | |||
| 42633748db | |||
| cd22feecf8 | |||
| 20584dc08f | |||
| 8e4eb591c1 | |||
| 1b721d76b1 | |||
| f413224b24 | |||
| b526cd7e55 | |||
| 3f8379d07d | |||
| 7350543f24 | |||
| 56bde5df31 | |||
| bd9cdcf379 | |||
| 7c72d14027 | |||
| ccd9767b7a | |||
| dae8193bd4 | |||
| 1de3bd6911 | |||
| de2ce5acb4 | |||
| a1a98617ca | |||
| b85c75996d | |||
| cc2497f52f | |||
| 1e5fb4b79a | |||
| 6f7c6fc560 | |||
| c0721164be | |||
| b1fb550d5c | |||
| a6318dbdaf | |||
| e39d7ddb0f | |||
| 659f724f4c | |||
| baaf67d82e | |||
| 2244adff07 | |||
| e3d841ffd1 | |||
| 751632d79e | |||
| 545a9e4a82 | |||
| f25ec31565 | |||
| 374feda237 | |||
| 6ef0161835 | |||
| 1a3dbbc9dd | |||
| f1355da72e | |||
| bbe521ffec | |||
| 7d75d61621 | |||
| 1f77708a30 | |||
| 84fc78d60f | |||
| a182fe8c15 | |||
| 819d1e80c6 | |||
| ffdf115bf2 | |||
| eb7c690ea9 | |||
| 8c08571cd9 | |||
| 2853fa1882 | |||
| 1672c73988 | |||
| 5476415016 | |||
| 67ac74471d | |||
| a1c1a18dc5 | |||
| eb2894233c | |||
| 52e382b697 | |||
| 0d435e11b1 | |||
| 2755a0b763 | |||
| 043e030a4a | |||
| 490c30f853 | |||
| 59749a2b85 | |||
| fc3cf97b81 | |||
| d45a8784fc | |||
| 6d7319380c | |||
| 14e6c5ec08 | |||
| 98d07210e7 | |||
| 6923630389 | |||
| 0c73c81919 | |||
| a46b640da3 | |||
| 71f9d2beff | |||
| cf414a6053 | |||
| 05b395e81d | |||
| 6a2d7ae808 | |||
| 685afb6749 | |||
| 8efa82126b | |||
| 6e6aeb6a9e | |||
| c9e00ae0a5 | |||
| 08e0698b7e | |||
| 135a6a7bb4 | |||
| 8b4d7048f8 | |||
| f4d4ea5c36 |
+2
-1
@@ -9,7 +9,7 @@ site/
|
||||
/.idea/
|
||||
/Minio.iml
|
||||
**/access.log
|
||||
build
|
||||
build/
|
||||
vendor/**/*.js
|
||||
vendor/**/*.json
|
||||
release
|
||||
@@ -23,3 +23,4 @@ parts/
|
||||
prime/
|
||||
snap/.snapcraft/
|
||||
stage/
|
||||
.sia_temp/
|
||||
|
||||
+1
-1
@@ -26,4 +26,4 @@ after_success:
|
||||
- bash <(curl -s https://codecov.io/bash)
|
||||
|
||||
go:
|
||||
- 1.9.1
|
||||
- 1.9.4
|
||||
|
||||
+4
-1
@@ -1,10 +1,13 @@
|
||||
FROM golang:1.9.1-alpine3.6
|
||||
FROM golang:1.9.4-alpine3.6
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
|
||||
|
||||
+4
-1
@@ -1,10 +1,13 @@
|
||||
FROM golang:1.9.1-alpine3.6
|
||||
FROM golang:1.9.4-alpine3.6
|
||||
|
||||
MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV PATH $PATH:$GOPATH/bin
|
||||
ENV CGO_ENABLED 0
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key
|
||||
|
||||
WORKDIR /go/src/github.com/minio/
|
||||
|
||||
|
||||
@@ -4,6 +4,10 @@ MAINTAINER Minio Inc <dev@minio.io>
|
||||
|
||||
COPY dockerscripts/docker-entrypoint.sh dockerscripts/healthcheck.sh /usr/bin/
|
||||
|
||||
ENV MINIO_UPDATE off
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key
|
||||
|
||||
RUN \
|
||||
apk add --no-cache ca-certificates && \
|
||||
apk add --no-cache --virtual .build-deps curl && \
|
||||
|
||||
@@ -58,15 +58,13 @@ spelling:
|
||||
check: test
|
||||
test: verifiers build
|
||||
@echo "Running unit tests"
|
||||
@go test $(GOFLAGS) .
|
||||
@go test $(GOFLAGS) github.com/minio/minio/cmd...
|
||||
@go test $(GOFLAGS) github.com/minio/minio/pkg...
|
||||
@go test $(GOFLAGS) ./...
|
||||
@echo "Verifying build"
|
||||
@(env bash $(PWD)/buildscripts/verify-build.sh)
|
||||
|
||||
coverage: build
|
||||
@echo "Running all coverage for minio"
|
||||
@./buildscripts/go-coverage.sh
|
||||
@(env bash $(PWD)/buildscripts/go-coverage.sh)
|
||||
|
||||
# Builds minio locally.
|
||||
build:
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# Minio Quickstart Guide
|
||||
[](https://slack.minio.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio) [](https://build.snapcraft.io/user/minio/minio)
|
||||
[](https://app.fossa.io/projects/git%2Bgithub.com%2Fminio%2Fminio?ref=badge_shield)
|
||||
|
||||
Minio is an object storage server released under Apache License v2.0. It is compatible with Amazon S3 cloud storage service. It is best suited for storing unstructured data such as photos, videos, log files, backups and container / VM images. Size of an object can range from a few KBs to a maximum of 5TB.
|
||||
|
||||
@@ -26,8 +27,8 @@ Install minio packages using [Homebrew](http://brew.sh/)
|
||||
brew install minio/stable/minio
|
||||
minio server /data
|
||||
```
|
||||
#### Note
|
||||
If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
|
||||
|
||||
> NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
|
||||
```sh
|
||||
brew uninstall minio
|
||||
brew install minio/stable/minio
|
||||
@@ -53,18 +54,15 @@ chmod +x minio
|
||||
```
|
||||
|
||||
### Snap
|
||||
You can install the latest `minio` [snap](https://snapcraft.io), and help testing the most recent changes of the master branch in [all the supported Linux distros](https://snapcraft.io/docs/core/install) with:
|
||||
|
||||
Install minio using [Snap](https://snapcraft.io)
|
||||
```sh
|
||||
sudo snap install minio --edge
|
||||
```
|
||||
|
||||
Every time a new version of `minio` is pushed to the store, you will get it updated automatically.
|
||||
|
||||
You will need to allow the minio snap to observe mounts in the system:
|
||||
|
||||
Start minio using `snap run` command
|
||||
```sh
|
||||
sudo snap connect minio:mount-observe
|
||||
sudo snap run minio server /data
|
||||
```
|
||||
|
||||
## Microsoft Windows
|
||||
@@ -88,7 +86,6 @@ service minio start
|
||||
```
|
||||
|
||||
## Install from Source
|
||||
|
||||
Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
|
||||
|
||||
```sh
|
||||
@@ -103,6 +100,11 @@ Minio Server comes with an embedded web based object browser. Point your web bro
|
||||
## Test using Minio Client `mc`
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the Minio Client [Quickstart Guide](https://docs.minio.io/docs/minio-client-quickstart-guide) for further instructions.
|
||||
|
||||
## Pre-existing data
|
||||
When deployed on a single drive, Minio server lets clients access any pre-existing data in the data directory. For example, if Minio is started with the command `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
|
||||
|
||||
The above statement is also valid for all gateway backends.
|
||||
|
||||
## Explore Further
|
||||
- [Minio Erasure Code QuickStart Guide](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
|
||||
- [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
|
||||
@@ -113,3 +115,7 @@ Minio Server comes with an embedded web based object browser. Point your web bro
|
||||
|
||||
## Contribute to Minio Project
|
||||
Please follow Minio [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
|
||||
|
||||
|
||||
## License
|
||||
[](https://app.fossa.io/projects/git%2Bgithub.com%2Fminio%2Fminio?ref=badge_large)
|
||||
-197
@@ -1,197 +0,0 @@
|
||||
# Minio 快速入门 [](https://slack.minio.io) [](https://goreportcard.com/report/minio/minio) [](https://codecov.io/gh/minio/minio) [](https://build.snapcraft.io/user/minio/minio)
|
||||
|
||||
Minio是一个对象存储服务,基于Apache License v2.0协议. 它完全兼容亚马逊的S3云储存服务,非常适合于存储很多非结构化的数据,例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等,而一个对象文件可以是任意大小,从几kb到最大5T不等。
|
||||
|
||||
## 1. 下载
|
||||
|
||||
Minio是一个非常轻量的服务,可以很简单的和其他应用的结合,类似 NodeJS, Redis 或者 MySQL.
|
||||
|
||||
| Platform| Architecture | URL|
|
||||
| ----------| -------- | ------|
|
||||
|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio|
|
||||
|Apple OS X|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio|
|
||||
|Microsoft Windows|64-bit|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe|
|
||||
|FreeBSD|64-bit|https://dl.minio.io/server/minio/release/freebsd-amd64/minio|
|
||||
|
||||
### Homebrew 安装
|
||||
|
||||
使用[Homebrew](http://brew.sh/) 来安装minio
|
||||
|
||||
```sh
|
||||
$ brew install minio
|
||||
$ minio --help
|
||||
```
|
||||
|
||||
### 源码安装
|
||||
|
||||
源码安装只针对开发者和一些高级用户,如果你还没有golang的环境,请安装golang官网安装[How to install Golang](https://docs.minio.io/docs/zh-CN/how-to-install-golang).
|
||||
|
||||
```sh
|
||||
|
||||
$ go get -u github.com/minio/minio
|
||||
|
||||
|
||||
```
|
||||
|
||||
## 2. 运行Minio服务
|
||||
|
||||
|
||||
### GNU/Linux
|
||||
|
||||
```sh
|
||||
|
||||
$ chmod +x minio
|
||||
$ ./minio --help
|
||||
$ ./minio server /data
|
||||
|
||||
端点: http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
AccessKey: USWUXHGYZQYFYFFIT3RE
|
||||
SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
区域: us-east-1
|
||||
|
||||
浏览器访问入口:
|
||||
http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
|
||||
命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
|
||||
$ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
|
||||
对象操作API (兼容Amazon S3):
|
||||
Go: https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
|
||||
Java: https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
|
||||
Python: https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
|
||||
JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
|
||||
|
||||
```
|
||||
|
||||
### OS X
|
||||
|
||||
|
||||
```sh
|
||||
|
||||
$ chmod 755 minio
|
||||
$ ./minio --help
|
||||
$ ./minio server /data
|
||||
|
||||
端点: http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
AccessKey: USWUXHGYZQYFYFFIT3RE
|
||||
SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
区域: us-east-1
|
||||
|
||||
浏览器访问入口:
|
||||
http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
|
||||
命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
|
||||
$ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
|
||||
对象操作API (兼容Amazon S3):
|
||||
Go: https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
|
||||
Java: https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
|
||||
Python: https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
|
||||
JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
|
||||
|
||||
```
|
||||
|
||||
### Microsoft Windows
|
||||
|
||||
```sh
|
||||
|
||||
C:\Users\Username\Downloads> minio.exe --help
|
||||
C:\Users\Username\Downloads> minio.exe server D:\Photos
|
||||
|
||||
端点: http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
AccessKey: USWUXHGYZQYFYFFIT3RE
|
||||
SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
区域: us-east-1
|
||||
|
||||
浏览器访问入口:
|
||||
http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
|
||||
命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
|
||||
$ mc.exe config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
|
||||
对象操作API (兼容Amazon S3):
|
||||
Go: https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
|
||||
Java: https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
|
||||
Python: https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
|
||||
JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
|
||||
|
||||
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
```sh
|
||||
|
||||
$ docker pull minio/minio
|
||||
$ docker run -p 9000:9000 minio/minio
|
||||
|
||||
```
|
||||
访问minio的docker入门指南获得更多内容 [here](https://docs.minio.io/docs/zh-CN/minio-docker-quickstart-guide)
|
||||
|
||||
### FreeBSD
|
||||
|
||||
```sh
|
||||
|
||||
$ chmod 755 minio
|
||||
$ ./minio --help
|
||||
$ ./minio server /data
|
||||
|
||||
端点: http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
AccessKey: USWUXHGYZQYFYFFIT3RE
|
||||
SecretKey: MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
区域: us-east-1
|
||||
|
||||
浏览器访问入口:
|
||||
http://10.0.0.10:9000 http://127.0.0.1:9000 http://172.17.0.1:9000
|
||||
|
||||
命令行访问: https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide
|
||||
$ mc config host add myminio http://10.0.0.10:9000 USWUXHGYZQYFYFFIT3RE MOJRH0mkL1IPauahWITSVvyDrQbEEIwljvmxdq03
|
||||
|
||||
对象操作API (兼容Amazon S3):
|
||||
Go: https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide
|
||||
Java: https://docs.minio.io/docs/zh-CN/java-client-quickstart-guide
|
||||
Python: https://docs.minio.io/docs/zh-CN/python-client-quickstart-guide
|
||||
JavaScript: https://docs.minio.io/docs/zh-CN/javascript-client-quickstart-guide
|
||||
|
||||
|
||||
```
|
||||
请访问FreeBSD的官网指南获取更多详细信息[here](https://www.freebsd.org/doc/handbook/zfs-quickstart.html)
|
||||
|
||||
## 3. 使用浏览器测试minio服务
|
||||
|
||||
打开浏览器并输入 http://127.0.0.1:9000 查看在minio服务器上面的所有bucket
|
||||
|
||||

|
||||
|
||||
|
||||
## 4. 使用`mc`测试minio服务
|
||||
|
||||
|
||||
按照 [这个](https://docs.minio.io/docs/minio-client-quickstart-guide) 安装mc. 使用 `mc ls` 命令显示所有在minio服务上面的bucket.
|
||||
|
||||
```sh
|
||||
|
||||
$ mc ls myminio/
|
||||
[2015-08-05 08:13:22 IST] 0B andoria/
|
||||
[2015-08-05 06:14:26 IST] 0B deflector/
|
||||
[2015-08-05 08:13:11 IST] 0B ferenginar/
|
||||
[2016-03-08 14:56:35 IST] 0B jarjarbing/
|
||||
[2016-01-20 16:07:41 IST] 0B my.minio.io/
|
||||
|
||||
```
|
||||
|
||||
查看更多的例子请访问 [Minio Client Complete Guide](https://docs.minio.io/docs/zh-CN/minio-client-complete-guide).
|
||||
|
||||
|
||||
## 5. 更多内容
|
||||
|
||||
- [Minio Erasure Code 快速入门](https://docs.minio.io/docs/zh-CN/minio-erasure-code-quickstart-guide)
|
||||
- [Minio Docker 快速入门](https://docs.minio.io/docs/zh-CN/minio-docker-quickstart-guide)
|
||||
- [使用`mc`测试 Minio Server](https://docs.minio.io/docs/zh-CN/minio-client-quickstart-guide)
|
||||
- [使用 `aws-cli` 测试 Minio Server](https://docs.minio.io/docs/zh-CN/aws-cli-with-minio)
|
||||
- [使用 `s3cmd` 测试 Minio Server](https://docs.minio.io/docs/zh-CN/s3cmd-with-minio)
|
||||
- [使用 `minio-go` SDK ce's测试 Minio Server](https://docs.minio.io/docs/zh-CN/golang-client-quickstart-guide)
|
||||
|
||||
|
||||
## 6. 给Minio项目贡献
|
||||
请按照Minio [贡献者指导手册](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
|
||||
+121
@@ -0,0 +1,121 @@
|
||||
# Minio Quickstart Guide [](https://slack.minio.io) [](https://goreportcard.com/report/minio/minio) [](https://hub.docker.com/r/minio/minio/) [](https://codecov.io/gh/minio/minio)
|
||||
|
||||
Minio 是一个基于Apache License v2.0开源协议的对象存储服务。它兼容亚马逊S3云存储服务接口,非常适合于存储大容量非结构化的数据,例如图片、视频、日志文件、备份数据和容器/虚拟机镜像等,而一个对象文件可以是任意大小,从几kb到最大5T不等。
|
||||
|
||||
Minio是一个非常轻量的服务,可以很简单的和其他应用的结合,类似 NodeJS, Redis 或者 MySQL。
|
||||
|
||||
## Docker 容器
|
||||
### 稳定版
|
||||
```
|
||||
docker pull minio/minio
|
||||
docker run -p 9000:9000 minio/minio server /data
|
||||
```
|
||||
|
||||
### 尝鲜版
|
||||
```
|
||||
docker pull minio/minio:edge
|
||||
docker run -p 9000:9000 minio/minio:edge server /data
|
||||
```
|
||||
更多Docker部署信息请访问 [这里](https://docs.minio.io/docs/minio-docker-quickstart-guide)
|
||||
|
||||
## macOS
|
||||
### Homebrew
|
||||
使用 [Homebrew](http://brew.sh/)安装minio
|
||||
|
||||
```sh
|
||||
brew install minio/stable/minio
|
||||
minio server /data
|
||||
```
|
||||
#### Note
|
||||
如果你之前使用 `brew install minio`安装过minio, 可以用 `minio/stable/minio` 官方镜像进行重装. 由于golang 1.8的bug,homebrew版本不太稳定。
|
||||
|
||||
```
|
||||
brew uninstall minio
|
||||
brew install minio/stable/minio
|
||||
```
|
||||
|
||||
### 下载二进制文件
|
||||
| 操作系统| CPU架构 | 地址|
|
||||
| ----------| -------- | ------|
|
||||
|Apple macOS|64-bit Intel|https://dl.minio.io/server/minio/release/darwin-amd64/minio |
|
||||
```sh
|
||||
chmod 755 minio
|
||||
./minio server /data
|
||||
```
|
||||
|
||||
## GNU/Linux
|
||||
### 下载二进制文件
|
||||
| 操作系统| CPU架构 | 地址|
|
||||
| ----------| -------- | ------|
|
||||
|GNU/Linux|64-bit Intel|https://dl.minio.io/server/minio/release/linux-amd64/minio |
|
||||
```sh
|
||||
chmod +x minio
|
||||
./minio server /data
|
||||
```
|
||||
|
||||
### 快照版
|
||||
你可以下载最新版 `minio` [快照](https://snapcraft.io), 并且帮助我们一起验证master分支上[所有支持的Linux发行版](https://snapcraft.io/docs/core/install) 的一些最新修改:
|
||||
|
||||
```sh
|
||||
sudo snap install minio --edge
|
||||
```
|
||||
|
||||
每次有最新的 `minio` 被推送到服务器,你都会自动更新下来.
|
||||
|
||||
你需要允许minio snap来观察其安装:
|
||||
|
||||
```sh
|
||||
sudo snap connect minio:mount-observe
|
||||
```
|
||||
|
||||
## 微软Windows系统
|
||||
### 下载二进制文件
|
||||
| 操作系统| CPU架构 | 地址|
|
||||
| ----------| -------- | ------|
|
||||
|微软Windows系统|64位|https://dl.minio.io/server/minio/release/windows-amd64/minio.exe |
|
||||
```sh
|
||||
minio.exe server D:\Photos
|
||||
```
|
||||
|
||||
## FreeBSD
|
||||
### Port
|
||||
使用 [pkg](https://github.com/freebsd/pkg)进行安装。
|
||||
|
||||
```sh
|
||||
pkg install minio
|
||||
sysrc minio_enable=yes
|
||||
sysrc minio_disks=/home/user/Photos
|
||||
service minio start
|
||||
```
|
||||
|
||||
## 使用源码安装
|
||||
|
||||
采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境, 请参考 [How to install Golang](https://docs.minio.io/docs/how-to-install-golang).
|
||||
|
||||
```sh
|
||||
go get -u github.com/minio/minio
|
||||
```
|
||||
|
||||
## 使用Minio浏览器进行验证
|
||||
安装后使用浏览器访问[http://127.0.0.1:9000](http://127.0.0.1:9000),如果可以访问,则表示minio已经安装成功。
|
||||
|
||||

|
||||
|
||||
## 使用Minio客户端 `mc`进行验证
|
||||
`mc` 提供了一些UNIX常用命令的替代品,像ls, cat, cp, mirror, diff这些。 它支持文件系统和亚马逊S3云存储服务。 更多信息请参考 [mc快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide) 。
|
||||
|
||||
## 已经存在的数据
|
||||
当在单块磁盘上部署Minio server,Minio server允许客户端访问数据目录下已经存在的数据。比如,如果Minio使用`minio server /mnt/data`启动,那么所有已经在`/mnt/data`目录下的数据都可以被客户端访问到。
|
||||
|
||||
上述描述对所有网关后端同样有效。
|
||||
|
||||
## 了解更多
|
||||
- [Minio纠删码入门](https://docs.minio.io/docs/minio-erasure-code-quickstart-guide)
|
||||
- [`mc`快速入门](https://docs.minio.io/docs/minio-client-quickstart-guide)
|
||||
- [使用 `aws-cli`](https://docs.minio.io/docs/aws-cli-with-minio)
|
||||
- [使用 `s3cmd`](https://docs.minio.io/docs/s3cmd-with-minio)
|
||||
- [使用 `minio-go` SDK](https://docs.minio.io/docs/golang-client-quickstart-guide)
|
||||
- [Minio文档](https://docs.minio.io)
|
||||
|
||||
## 如何参与到Minio项目
|
||||
请参考 [贡献者指南](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)。欢迎各位中国程序员加到Minio项目中。
|
||||
+8
-11
@@ -38,21 +38,18 @@ test_script:
|
||||
# Unit tests
|
||||
- ps: Add-AppveyorTest "Unit Tests" -Outcome Running
|
||||
- mkdir build\coverage
|
||||
- go test -v -timeout 17m -race github.com/minio/minio/cmd...
|
||||
- go test -v -race github.com/minio/minio/pkg...
|
||||
# FIXME(aead): enable codecov after issue https://github.com/golang/go/issues/18468 is solved.
|
||||
# - go test -v -timeout 17m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
|
||||
- for /f "" %%G in ('go list github.com/minio/minio/... ^| find /i /v "browser/"') do ( go test -v -timeout 20m -race %%G )
|
||||
- go test -v -timeout 20m -coverprofile=build\coverage\coverage.txt -covermode=atomic github.com/minio/minio/cmd
|
||||
- ps: Update-AppveyorTest "Unit Tests" -Outcome Passed
|
||||
|
||||
after_test:
|
||||
# FIXME(aead): enable codecov after issue https://github.com/golang/go/issues/18468 is solved.
|
||||
# - go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
|
||||
# - ps: Push-AppveyorArtifact build\coverage\coverage.txt
|
||||
# - ps: Push-AppveyorArtifact build\coverage\coverage.html
|
||||
- go tool cover -html=build\coverage\coverage.txt -o build\coverage\coverage.html
|
||||
- ps: Push-AppveyorArtifact build\coverage\coverage.txt
|
||||
- ps: Push-AppveyorArtifact build\coverage\coverage.html
|
||||
# Upload coverage report.
|
||||
# - "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
|
||||
# - pip install codecov
|
||||
# - codecov -X gcov -f "build\coverage\coverage.txt"
|
||||
- "SET PATH=C:\\Python34;C:\\Python34\\Scripts;%PATH%"
|
||||
- pip install codecov
|
||||
- codecov -X gcov -f "build\coverage\coverage.txt"
|
||||
|
||||
# to disable deployment
|
||||
deploy: off
|
||||
|
||||
@@ -41,6 +41,8 @@ import _Login from './js/components/Login.js'
|
||||
import _Browse from './js/components/Browse.js'
|
||||
import fontAwesome from 'font-awesome/css/font-awesome.css'
|
||||
|
||||
import MaterialDesignIconicFonts from 'material-design-iconic-font/dist/css/material-design-iconic-font.min.css'
|
||||
|
||||
import Web from './js/web'
|
||||
window.Web = Web
|
||||
|
||||
|
||||
@@ -24,6 +24,8 @@ export const SET_CURRENT_BUCKET = 'SET_CURRENT_BUCKET'
|
||||
export const SET_CURRENT_PATH = 'SET_CURRENT_PATH'
|
||||
export const SET_BUCKETS = 'SET_BUCKETS'
|
||||
export const ADD_BUCKET = 'ADD_BUCKET'
|
||||
export const REMOVE_BUCKET = 'REMOVE_BUCKET'
|
||||
export const SHOW_BUCKET_DROPDOWN = 'SHOW_BUCKET_DROPDOWN'
|
||||
export const SET_VISIBLE_BUCKETS = 'SET_VISIBLE_BUCKETS'
|
||||
export const SET_OBJECTS = 'SET_OBJECTS'
|
||||
export const APPEND_OBJECTS = 'APPEND_OBJECTS'
|
||||
@@ -173,6 +175,27 @@ export const addBucket = bucket => {
|
||||
}
|
||||
}
|
||||
|
||||
export const removeBucket = bucket => {
|
||||
return {
|
||||
type: REMOVE_BUCKET,
|
||||
bucket
|
||||
}
|
||||
}
|
||||
|
||||
export const showBucketDropdown = bucket => {
|
||||
return {
|
||||
type: SHOW_BUCKET_DROPDOWN,
|
||||
showBucketDropdown: true
|
||||
}
|
||||
}
|
||||
|
||||
export const hideBucketDropdown = bucket => {
|
||||
return {
|
||||
type: SHOW_BUCKET_DROPDOWN,
|
||||
showBucketDropdown: false
|
||||
}
|
||||
}
|
||||
|
||||
export const showMakeBucketModal = () => {
|
||||
return {
|
||||
type: SHOW_MAKEBUCKET_MODAL,
|
||||
@@ -314,10 +337,35 @@ export const selectBucket = (newCurrentBucket, prefix) => {
|
||||
}
|
||||
}
|
||||
|
||||
export const deleteBucket = (bucket) => {
|
||||
return (dispatch, getState) => {
|
||||
// DeleteBucket() RPC call will ONLY delete a bucket if it is empty of
|
||||
// objects. This means a call can just be sent, as it is entirely reversable
|
||||
// and won't do any permanent damage.
|
||||
web.DeleteBucket({
|
||||
bucketName: bucket
|
||||
})
|
||||
.then(() => {
|
||||
dispatch(showAlert({
|
||||
type: 'info',
|
||||
message: `Bucket '${bucket}' has been deleted.`
|
||||
}))
|
||||
dispatch(removeBucket(bucket))
|
||||
})
|
||||
.catch(err => {
|
||||
let message = err.message
|
||||
dispatch(showAlert({
|
||||
type: 'danger',
|
||||
message: message
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
export const listObjects = () => {
|
||||
return (dispatch, getState) => {
|
||||
const {currentBucket, currentPath, marker, objects, istruncated, web} = getState()
|
||||
if (!istruncated) return
|
||||
const {buckets, currentBucket, currentPath, marker, objects, istruncated, web} = getState()
|
||||
if (!istruncated || buckets.length === 0) return
|
||||
web.ListObjects({
|
||||
bucketName: currentBucket,
|
||||
prefix: currentPath,
|
||||
@@ -352,7 +400,6 @@ export const listObjects = () => {
|
||||
export const selectPrefix = prefix => {
|
||||
return (dispatch, getState) => {
|
||||
const {currentBucket, web} = getState()
|
||||
dispatch(resetObjects())
|
||||
dispatch(setLoadPath(prefix))
|
||||
web.ListObjects({
|
||||
bucketName: currentBucket,
|
||||
@@ -367,6 +414,7 @@ export const selectPrefix = prefix => {
|
||||
object.name = object.name.replace(`${prefix}`, '');
|
||||
return object
|
||||
})
|
||||
dispatch(resetObjects())
|
||||
dispatch(appendObjects(
|
||||
objects,
|
||||
res.nextmarker,
|
||||
|
||||
@@ -210,9 +210,19 @@ export default class Browse extends React.Component {
|
||||
dispatch(actions.hideAbout())
|
||||
}
|
||||
|
||||
toggleBucketDropdown(e) {
|
||||
const {dispatch, showBucketDropdown} = this.props
|
||||
if (showBucketDropdown) {
|
||||
dispatch(actions.hideBucketDropdown())
|
||||
} else {
|
||||
dispatch(actions.showBucketDropdown())
|
||||
}
|
||||
}
|
||||
|
||||
showBucketPolicy(e) {
|
||||
e.preventDefault()
|
||||
const {dispatch} = this.props
|
||||
this.toggleBucketDropdown(e)
|
||||
dispatch(actions.showBucketPolicy())
|
||||
}
|
||||
|
||||
@@ -222,10 +232,17 @@ export default class Browse extends React.Component {
|
||||
dispatch(actions.hideBucketPolicy())
|
||||
}
|
||||
|
||||
deleteBucket(e, bucket) {
|
||||
e.preventDefault()
|
||||
const {dispatch} = this.props
|
||||
this.toggleBucketDropdown(e)
|
||||
dispatch(actions.deleteBucket(bucket))
|
||||
browserHistory.push(`${minioBrowserPrefix}/`)
|
||||
}
|
||||
|
||||
uploadFile(e) {
|
||||
e.preventDefault()
|
||||
const {dispatch, buckets} = this.props
|
||||
|
||||
const {dispatch, buckets, currentBucket} = this.props
|
||||
if (buckets.length === 0) {
|
||||
dispatch(actions.showAlert({
|
||||
type: 'danger',
|
||||
@@ -233,6 +250,13 @@ export default class Browse extends React.Component {
|
||||
}))
|
||||
return
|
||||
}
|
||||
if (currentBucket === '') {
|
||||
dispatch(actions.showAlert({
|
||||
type: 'danger',
|
||||
message: "Please choose a bucket before trying to upload files."
|
||||
}))
|
||||
return
|
||||
}
|
||||
let file = e.target.files[0]
|
||||
e.target.value = null
|
||||
this.xhr = new XMLHttpRequest()
|
||||
@@ -577,7 +601,9 @@ export default class Browse extends React.Component {
|
||||
<SideBar searchBuckets={ this.searchBuckets.bind(this) }
|
||||
selectBucket={ this.selectBucket.bind(this) }
|
||||
clickOutside={ this.hideSidebar.bind(this) }
|
||||
showPolicy={ this.showBucketPolicy.bind(this) } />
|
||||
showPolicy={ this.showBucketPolicy.bind(this) }
|
||||
deleteBucket={ this.deleteBucket.bind(this) }
|
||||
toggleBucketDropdown={ this.toggleBucketDropdown.bind(this) } />
|
||||
<div className="fe-body">
|
||||
<div className={ 'list-actions' + (classNames({
|
||||
' list-actions-toggled': checkedObjects.length > 0
|
||||
|
||||
@@ -21,8 +21,9 @@ import Scrollbars from 'react-custom-scrollbars/lib/Scrollbars'
|
||||
import connect from 'react-redux/lib/components/connect'
|
||||
|
||||
import logo from '../../img/logo.svg'
|
||||
import Dropdown from 'react-bootstrap/lib/Dropdown'
|
||||
|
||||
let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy}) => {
|
||||
let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchBuckets, sidebarStatus, clickOutside, showPolicy, deleteBucket, toggleBucketDropdown, showBucketDropdown}) => {
|
||||
|
||||
const list = visibleBuckets.map((bucket, i) => {
|
||||
return <li className={ classNames({
|
||||
@@ -33,7 +34,19 @@ let SideBar = ({visibleBuckets, loadBucket, currentBucket, selectBucket, searchB
|
||||
}) }>
|
||||
{ bucket }
|
||||
</a>
|
||||
<i className="fesli-trigger" onClick={ showPolicy }></i>
|
||||
<Dropdown open={bucket === currentBucket && showBucketDropdown} onToggle={toggleBucketDropdown} className="bucket-dropdown" id="bucket-dropdown">
|
||||
<Dropdown.Toggle noCaret>
|
||||
<i className="zmdi zmdi-more-vert" />
|
||||
</Dropdown.Toggle>
|
||||
<Dropdown.Menu className="dropdown-menu-right">
|
||||
<li>
|
||||
<a onClick={ showPolicy }>Edit policy</a>
|
||||
</li>
|
||||
<li>
|
||||
<a onClick={ (e) => deleteBucket(e, bucket) }>Delete</a>
|
||||
</li>
|
||||
</Dropdown.Menu>
|
||||
</Dropdown>
|
||||
</li>
|
||||
})
|
||||
|
||||
@@ -80,6 +93,7 @@ export default connect(state => {
|
||||
visibleBuckets: state.visibleBuckets,
|
||||
loadBucket: state.loadBucket,
|
||||
currentBucket: state.currentBucket,
|
||||
showBucketDropdown: state.showBucketDropdown,
|
||||
sidebarStatus: state.sidebarStatus
|
||||
}
|
||||
})(SideBar)
|
||||
|
||||
@@ -25,6 +25,7 @@ export default (state = {
|
||||
storageInfo: {},
|
||||
serverInfo: {},
|
||||
currentBucket: '',
|
||||
showBucketDropdown: false,
|
||||
currentPath: '',
|
||||
showMakeBucketModal: false,
|
||||
uploads: {},
|
||||
@@ -71,6 +72,14 @@ export default (state = {
|
||||
newState.buckets = [action.bucket, ...newState.buckets]
|
||||
newState.visibleBuckets = [action.bucket, ...newState.visibleBuckets]
|
||||
break
|
||||
case actions.REMOVE_BUCKET:
|
||||
newState.buckets = newState.buckets.filter(bucket => bucket != action.bucket)
|
||||
newState.visibleBuckets = newState.visibleBuckets.filter(bucket => bucket != action.bucket)
|
||||
newState.currentBucket = ""
|
||||
break
|
||||
case actions.SHOW_BUCKET_DROPDOWN:
|
||||
newState.showBucketDropdown = action.showBucketDropdown
|
||||
break
|
||||
case actions.SET_VISIBLE_BUCKETS:
|
||||
newState.visibleBuckets = action.visibleBuckets
|
||||
break
|
||||
|
||||
@@ -87,6 +87,9 @@ export default class Web {
|
||||
MakeBucket(args) {
|
||||
return this.makeCall('MakeBucket', args)
|
||||
}
|
||||
DeleteBucket(args) {
|
||||
return this.makeCall('DeleteBucket', args)
|
||||
}
|
||||
ListObjects(args) {
|
||||
return this.makeCall('ListObjects', args)
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
.fe-body {
|
||||
@media(min-width: @screen-md-min) {
|
||||
padding: 0 0 40px @fe-sidebar-width;
|
||||
padding: 0 0 80px @fe-sidebar-width;
|
||||
}
|
||||
|
||||
@media(max-width: @screen-sm-max) {
|
||||
@@ -79,7 +79,7 @@
|
||||
text-align: center;
|
||||
border: 0;
|
||||
padding: 0;
|
||||
|
||||
|
||||
span {
|
||||
display: inline-block;
|
||||
height: 100%;
|
||||
|
||||
@@ -108,7 +108,7 @@ div.fesl-row {
|
||||
&[data-type=zip] { .list-type(#427089, '\f1c6'); }
|
||||
&[data-type=audio] { .list-type(#009688, '\f1c7'); }
|
||||
&[data-type=code] { .list-type(#997867, "\f1c9"); }
|
||||
&[data-type=excel] { .list-type(#f1c3, '\f1c3'); }
|
||||
&[data-type=excel] { .list-type(#64c866, '\f1c3'); }
|
||||
&[data-type=image] { .list-type(#f06292, '\f1c5'); }
|
||||
&[data-type=video] { .list-type(#f8c363, '\f1c8'); }
|
||||
&[data-type=other] { .list-type(#afafaf, '\f016'); }
|
||||
@@ -159,7 +159,7 @@ div.fesl-row {
|
||||
top: 0;
|
||||
width: 35px;
|
||||
height: 35px;
|
||||
z-index: 20;
|
||||
z-index: 8;
|
||||
opacity: 0;
|
||||
cursor: pointer;
|
||||
|
||||
@@ -223,7 +223,7 @@ div.fesl-row {
|
||||
width: 15px;
|
||||
height: 15px;
|
||||
border: 2px solid @white;
|
||||
z-index: 10;
|
||||
z-index: 7;
|
||||
border-radius: 2px;
|
||||
top: 10px;
|
||||
left: 10px;
|
||||
|
||||
@@ -111,7 +111,7 @@
|
||||
}
|
||||
|
||||
&:hover {
|
||||
.fesli-trigger {
|
||||
.bucket-dropdown .dropdown-toggle {
|
||||
.opacity(0.6);
|
||||
|
||||
&:hover {
|
||||
@@ -132,17 +132,40 @@
|
||||
}
|
||||
}
|
||||
|
||||
.fesli-trigger {
|
||||
.opacity(0);
|
||||
.transition(all);
|
||||
.transition-duration(200ms);
|
||||
/* Dropdown */
|
||||
.bucket-dropdown {
|
||||
position: absolute;
|
||||
top: 0;
|
||||
right: 0;
|
||||
top: 0px;
|
||||
right: 0px;
|
||||
width: 35px;
|
||||
height: 100%;
|
||||
cursor: pointer;
|
||||
background: url(../../img/more-h-light.svg) no-repeat top 20px left;
|
||||
color: @white;
|
||||
|
||||
.dropdown-toggle {
|
||||
.opacity(0);
|
||||
.transition(all);
|
||||
.transition-duration(200ms);
|
||||
font-size: 20px;
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
.dropdown-menu-right {
|
||||
padding: 15px 0;
|
||||
margin-top: -1px;
|
||||
& li {
|
||||
a:before {
|
||||
content: none;
|
||||
-webkit-box-shadow: @dropdown-shadow;
|
||||
box-shadow: @dropdown-shadow;
|
||||
}
|
||||
&:not(.active):hover {
|
||||
& > a {
|
||||
color: @dropdown-link-hover-color;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Scrollbar */
|
||||
|
||||
@@ -60,6 +60,7 @@
|
||||
"humanize": "0.0.9",
|
||||
"json-loader": "^0.5.4",
|
||||
"local-storage-fallback": "^1.3.0",
|
||||
"material-design-iconic-font": "^2.2.0",
|
||||
"mime-db": "^1.25.0",
|
||||
"mime-types": "^2.1.13",
|
||||
"moment": "^2.15.1",
|
||||
|
||||
+61
-54
File diff suppressed because one or more lines are too long
@@ -21,7 +21,7 @@ _init() {
|
||||
|
||||
## Minimum required versions for build dependencies
|
||||
GIT_VERSION="1.0"
|
||||
GO_VERSION="1.9.1"
|
||||
GO_VERSION="1.9.4"
|
||||
OSX_VERSION="10.8"
|
||||
KNAME=$(uname -s)
|
||||
ARCH=$(uname -m)
|
||||
|
||||
@@ -3,8 +3,8 @@
|
||||
set -e
|
||||
echo "" > coverage.txt
|
||||
|
||||
for d in $(go list ./... | grep -v vendor); do
|
||||
go test -coverprofile=profile.out -covermode=atomic $d
|
||||
for d in $(go list ./... | grep -v browser); do
|
||||
go test -coverprofile=profile.out -covermode=atomic "$d"
|
||||
if [ -f profile.out ]; then
|
||||
cat profile.out >> coverage.txt
|
||||
rm profile.out
|
||||
|
||||
+367
-586
File diff suppressed because it is too large
Load Diff
+358
-795
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,690 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
// healStatusSummary - overall short summary of a healing sequence
|
||||
type healStatusSummary string
|
||||
|
||||
// healStatusSummary constants
|
||||
const (
|
||||
healNotStartedStatus healStatusSummary = "not started"
|
||||
healRunningStatus = "running"
|
||||
healStoppedStatus = "stopped"
|
||||
healFinishedStatus = "finished"
|
||||
)
|
||||
|
||||
const (
|
||||
// a heal sequence with this many un-consumed heal result
|
||||
// items blocks until heal-status consumption resumes or is
|
||||
// aborted due to timeout.
|
||||
maxUnconsumedHealResultItems = 1000
|
||||
|
||||
// if no heal-results are consumed (via the heal-status API)
|
||||
// for this timeout duration, the heal sequence is aborted.
|
||||
healUnconsumedTimeout = 24 * time.Hour
|
||||
|
||||
// time-duration to keep heal sequence state after it
|
||||
// completes.
|
||||
keepHealSeqStateDuration = time.Minute * 10
|
||||
)
|
||||
|
||||
var (
|
||||
errHealIdleTimeout = fmt.Errorf("healing results were not consumed for too long")
|
||||
errHealPushStopNDiscard = fmt.Errorf("heal push stopped due to heal stop signal")
|
||||
errHealStopSignalled = fmt.Errorf("heal stop signalled")
|
||||
|
||||
errFnHealFromAPIErr = func(err error) error {
|
||||
errCode := toAPIErrorCode(err)
|
||||
apiErr := getAPIError(errCode)
|
||||
return fmt.Errorf("Heal internal error: %s: %s",
|
||||
apiErr.Code, apiErr.Description)
|
||||
}
|
||||
)
|
||||
|
||||
// healSequenceStatus - accumulated status of the heal sequence
|
||||
type healSequenceStatus struct {
|
||||
// lock to update this structure as it is concurrently
|
||||
// accessed
|
||||
updateLock *sync.RWMutex
|
||||
|
||||
// summary and detail for failures
|
||||
Summary healStatusSummary `json:"Summary"`
|
||||
FailureDetail string `json:"Detail,omitempty"`
|
||||
StartTime time.Time `json:"StartTime"`
|
||||
|
||||
// disk information
|
||||
NumDisks int `json:"NumDisks"`
|
||||
|
||||
// settings for the heal sequence
|
||||
HealSettings madmin.HealOpts `json:"Settings"`
|
||||
|
||||
// slice of available heal result records
|
||||
Items []madmin.HealResultItem `json:"Items"`
|
||||
}
|
||||
|
||||
// structure to hold state of all heal sequences in server memory
|
||||
type allHealState struct {
|
||||
sync.Mutex
|
||||
|
||||
// map of heal path to heal sequence
|
||||
healSeqMap map[string]*healSequence
|
||||
}
|
||||
|
||||
var (
|
||||
// global server heal state
|
||||
globalAllHealState allHealState
|
||||
)
|
||||
|
||||
// initAllHealState - initialize healing apparatus
|
||||
func initAllHealState(isErasureMode bool) {
|
||||
if !isErasureMode {
|
||||
return
|
||||
}
|
||||
|
||||
globalAllHealState = allHealState{
|
||||
healSeqMap: make(map[string]*healSequence),
|
||||
}
|
||||
}
|
||||
|
||||
// getHealSequence - Retrieve a heal sequence by path. The second
|
||||
// argument returns if a heal sequence actually exists.
|
||||
func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) {
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
h, exists = ahs.healSeqMap[path]
|
||||
return h, exists
|
||||
}
|
||||
|
||||
// LaunchNewHealSequence - launches a background routine that performs
|
||||
// healing according to the healSequence argument. For each heal
|
||||
// sequence, state is stored in the `globalAllHealState`, which is a
|
||||
// map of the heal path to `healSequence` which holds state about the
|
||||
// heal sequence.
|
||||
//
|
||||
// Heal results are persisted in server memory for
|
||||
// `keepHealSeqStateDuration`. This function also launches a
|
||||
// background routine to clean up heal results after the
|
||||
// aforementioned duration.
|
||||
func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
|
||||
respBytes []byte, errCode APIErrorCode, errMsg string) {
|
||||
|
||||
existsAndLive := false
|
||||
he, exists := ahs.getHealSequence(h.path)
|
||||
if exists {
|
||||
if !he.hasEnded() || len(he.currentStatus.Items) > 0 {
|
||||
existsAndLive = true
|
||||
}
|
||||
}
|
||||
if existsAndLive {
|
||||
// A heal sequence exists on the given path.
|
||||
if h.forceStarted {
|
||||
// stop the running heal sequence - wait for
|
||||
// it to finish.
|
||||
he.stop()
|
||||
for !he.hasEnded() {
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
} else {
|
||||
errMsg = "Heal is already running on the given path " +
|
||||
"(use force-start option to stop and start afresh). " +
|
||||
fmt.Sprintf("The heal was started by IP %s at %s",
|
||||
h.clientAddress, h.startTime)
|
||||
|
||||
return nil, ErrHealAlreadyRunning, errMsg
|
||||
}
|
||||
}
|
||||
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
|
||||
// Check if new heal sequence to be started overlaps with any
|
||||
// existing, running sequence
|
||||
for k, hSeq := range ahs.healSeqMap {
|
||||
if !hSeq.hasEnded() && (strings.HasPrefix(k, h.path) ||
|
||||
strings.HasPrefix(h.path, k)) {
|
||||
|
||||
errMsg = "The provided heal sequence path overlaps with an existing " +
|
||||
fmt.Sprintf("heal path: %s", k)
|
||||
return nil, ErrHealOverlappingPaths, errMsg
|
||||
}
|
||||
}
|
||||
|
||||
// Add heal state and start sequence
|
||||
ahs.healSeqMap[h.path] = h
|
||||
|
||||
// Launch top-level background heal go-routine
|
||||
go h.healSequenceStart()
|
||||
|
||||
// Launch clean-up routine to remove this heal sequence (after
|
||||
// it ends) from the global state after timeout has elapsed.
|
||||
go func() {
|
||||
var keepStateTimeout <-chan time.Time
|
||||
ticker := time.NewTicker(time.Minute)
|
||||
defer ticker.Stop()
|
||||
everyMinute := ticker.C
|
||||
for {
|
||||
select {
|
||||
// Check every minute if heal sequence has ended.
|
||||
case <-everyMinute:
|
||||
if h.hasEnded() {
|
||||
keepStateTimeout = time.After(keepHealSeqStateDuration)
|
||||
everyMinute = nil
|
||||
}
|
||||
|
||||
// This case does not fire until the heal
|
||||
// sequence completes.
|
||||
case <-keepStateTimeout:
|
||||
// Heal sequence has ended, keep
|
||||
// results state duration has elapsed,
|
||||
// so purge state.
|
||||
ahs.Lock()
|
||||
defer ahs.Unlock()
|
||||
delete(ahs.healSeqMap, h.path)
|
||||
return
|
||||
|
||||
case <-globalServiceDoneCh:
|
||||
// server could be restarting - need
|
||||
// to exit immediately
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
h.clientToken,
|
||||
h.clientAddress,
|
||||
h.startTime,
|
||||
})
|
||||
if err != nil {
|
||||
errorIf(err, "Failed to marshal heal result into json.")
|
||||
return nil, ErrInternalError, ""
|
||||
}
|
||||
return b, ErrNone, ""
|
||||
}
|
||||
|
||||
// PopHealStatusJSON - Called by heal-status API. It fetches the heal
|
||||
// status results from global state and returns its JSON
|
||||
// representation. The clientToken helps ensure there aren't
|
||||
// conflicting clients fetching status.
|
||||
func (ahs *allHealState) PopHealStatusJSON(path string,
|
||||
clientToken string) ([]byte, APIErrorCode) {
|
||||
|
||||
// fetch heal state for given path
|
||||
h, exists := ahs.getHealSequence(path)
|
||||
if !exists {
|
||||
// If there is no such heal sequence, return error.
|
||||
return nil, ErrHealNoSuchProcess
|
||||
}
|
||||
|
||||
// Check if client-token is valid
|
||||
if clientToken != h.clientToken {
|
||||
return nil, ErrHealInvalidClientToken
|
||||
}
|
||||
|
||||
// Take lock to access and update the heal-sequence
|
||||
h.currentStatus.updateLock.Lock()
|
||||
defer h.currentStatus.updateLock.Unlock()
|
||||
|
||||
numItems := len(h.currentStatus.Items)
|
||||
|
||||
// calculate index of most recently available heal result
|
||||
// record.
|
||||
lastResultIndex := h.lastSentResultIndex
|
||||
if numItems > 0 {
|
||||
lastResultIndex = h.currentStatus.Items[numItems-1].ResultIndex
|
||||
}
|
||||
|
||||
// After sending status to client, and before relinquishing
|
||||
// the updateLock, reset Item to nil and record the result
|
||||
// index sent to the client.
|
||||
defer func(i int64) {
|
||||
h.lastSentResultIndex = i
|
||||
h.currentStatus.Items = nil
|
||||
}(lastResultIndex)
|
||||
|
||||
jbytes, err := json.Marshal(h.currentStatus)
|
||||
if err != nil {
|
||||
errorIf(err, "Failed to marshal heal result into json.")
|
||||
return nil, ErrInternalError
|
||||
}
|
||||
|
||||
return jbytes, ErrNone
|
||||
}
|
||||
|
||||
// healSequence - state for each heal sequence initiated on the
|
||||
// server.
|
||||
type healSequence struct {
|
||||
// bucket, and prefix on which heal seq. was initiated
|
||||
bucket, objPrefix string
|
||||
|
||||
// path is just bucket + "/" + objPrefix
|
||||
path string
|
||||
|
||||
// time at which heal sequence was started
|
||||
startTime time.Time
|
||||
|
||||
// Heal client info
|
||||
clientToken, clientAddress string
|
||||
|
||||
// was this heal sequence force started?
|
||||
forceStarted bool
|
||||
|
||||
// heal settings applied to this heal sequence
|
||||
settings madmin.HealOpts
|
||||
|
||||
// current accumulated status of the heal sequence
|
||||
currentStatus healSequenceStatus
|
||||
|
||||
// channel signalled by background routine when traversal has
|
||||
// completed
|
||||
traverseAndHealDoneCh chan error
|
||||
|
||||
// channel to signal heal sequence to stop (e.g. from the
|
||||
// heal-stop API)
|
||||
stopSignalCh chan struct{}
|
||||
|
||||
// the last result index sent to client
|
||||
lastSentResultIndex int64
|
||||
}
|
||||
|
||||
// NewHealSequence - creates healSettings, assumes bucket and
|
||||
// objPrefix are already validated.
|
||||
func newHealSequence(bucket, objPrefix, clientAddr string,
|
||||
numDisks int, hs madmin.HealOpts, forceStart bool) *healSequence {
|
||||
|
||||
return &healSequence{
|
||||
bucket: bucket,
|
||||
objPrefix: objPrefix,
|
||||
path: bucket + "/" + objPrefix,
|
||||
startTime: UTCNow(),
|
||||
clientToken: mustGetUUID(),
|
||||
clientAddress: clientAddr,
|
||||
forceStarted: forceStart,
|
||||
settings: hs,
|
||||
currentStatus: healSequenceStatus{
|
||||
Summary: healNotStartedStatus,
|
||||
HealSettings: hs,
|
||||
NumDisks: numDisks,
|
||||
updateLock: &sync.RWMutex{},
|
||||
},
|
||||
traverseAndHealDoneCh: make(chan error),
|
||||
stopSignalCh: make(chan struct{}),
|
||||
}
|
||||
}
|
||||
|
||||
// isQuitting - determines if the heal sequence is quitting (due to an
|
||||
// external signal)
|
||||
func (h *healSequence) isQuitting() bool {
|
||||
select {
|
||||
case <-h.stopSignalCh:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// check if the heal sequence has ended
|
||||
func (h *healSequence) hasEnded() bool {
|
||||
h.currentStatus.updateLock.RLock()
|
||||
summary := h.currentStatus.Summary
|
||||
h.currentStatus.updateLock.RUnlock()
|
||||
return summary == healStoppedStatus || summary == healFinishedStatus
|
||||
}
|
||||
|
||||
// stops the heal sequence - safe to call multiple times.
|
||||
func (h *healSequence) stop() {
|
||||
select {
|
||||
case <-h.stopSignalCh:
|
||||
default:
|
||||
close(h.stopSignalCh)
|
||||
}
|
||||
}
|
||||
|
||||
// pushHealResultItem - pushes a heal result item for consumption in
|
||||
// the heal-status API. It blocks if there are
|
||||
// maxUnconsumedHealResultItems. When it blocks, the heal sequence
|
||||
// routine is effectively paused - this happens when the server has
|
||||
// accumulated the maximum number of heal records per heal
|
||||
// sequence. When the client consumes further records, the heal
|
||||
// sequence automatically resumes. The return value indicates if the
|
||||
// operation succeeded.
|
||||
func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
|
||||
|
||||
// start a timer to keep an upper time limit to find an empty
|
||||
// slot to add the given heal result - if no slot is found it
|
||||
// means that the server is holding the maximum amount of
|
||||
// heal-results in memory and the client has not consumed it
|
||||
// for too long.
|
||||
unconsumedTimer := time.NewTimer(healUnconsumedTimeout)
|
||||
defer func() {
|
||||
// stop the timeout timer so it is garbage collected.
|
||||
if !unconsumedTimer.Stop() {
|
||||
<-unconsumedTimer.C
|
||||
}
|
||||
}()
|
||||
|
||||
var itemsLen int
|
||||
for {
|
||||
h.currentStatus.updateLock.Lock()
|
||||
itemsLen = len(h.currentStatus.Items)
|
||||
if itemsLen == maxUnconsumedHealResultItems {
|
||||
// unlock and wait to check again if we can push
|
||||
h.currentStatus.updateLock.Unlock()
|
||||
|
||||
// wait for a second, or quit if an external
|
||||
// stop signal is received or the
|
||||
// unconsumedTimer fires.
|
||||
select {
|
||||
// Check after a second
|
||||
case <-time.After(time.Second):
|
||||
continue
|
||||
|
||||
case <-h.stopSignalCh:
|
||||
// discard result and return.
|
||||
return errHealPushStopNDiscard
|
||||
|
||||
// Timeout if no results consumed for too
|
||||
// long.
|
||||
case <-unconsumedTimer.C:
|
||||
return errHealIdleTimeout
|
||||
|
||||
}
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Set the correct result index for the new result item
|
||||
if itemsLen > 0 {
|
||||
r.ResultIndex = 1 + h.currentStatus.Items[itemsLen-1].ResultIndex
|
||||
} else {
|
||||
r.ResultIndex = 1 + h.lastSentResultIndex
|
||||
}
|
||||
|
||||
// append to results
|
||||
h.currentStatus.Items = append(h.currentStatus.Items, r)
|
||||
|
||||
// release lock
|
||||
h.currentStatus.updateLock.Unlock()
|
||||
|
||||
// This is a "safe" point for the heal sequence to quit if
|
||||
// signalled externally.
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// healSequenceStart - this is the top-level background heal
|
||||
// routine. It launches another go-routine that actually traverses
|
||||
// on-disk data, checks and heals according to the selected
|
||||
// settings. This go-routine itself, (1) monitors the traversal
|
||||
// routine for completion, and (2) listens for external stop
|
||||
// signals. When either event happens, it sets the finish status for
|
||||
// the heal-sequence.
|
||||
func (h *healSequence) healSequenceStart() {
|
||||
// Set status as running
|
||||
h.currentStatus.updateLock.Lock()
|
||||
h.currentStatus.Summary = healRunningStatus
|
||||
h.currentStatus.StartTime = UTCNow()
|
||||
h.currentStatus.updateLock.Unlock()
|
||||
|
||||
go h.traverseAndHeal()
|
||||
|
||||
select {
|
||||
case err, ok := <-h.traverseAndHealDoneCh:
|
||||
h.currentStatus.updateLock.Lock()
|
||||
defer h.currentStatus.updateLock.Unlock()
|
||||
// Heal traversal is complete.
|
||||
if ok {
|
||||
// heal traversal had an error.
|
||||
h.currentStatus.Summary = healStoppedStatus
|
||||
h.currentStatus.FailureDetail = err.Error()
|
||||
} else {
|
||||
// heal traversal succeeded.
|
||||
h.currentStatus.Summary = healFinishedStatus
|
||||
}
|
||||
|
||||
case <-h.stopSignalCh:
|
||||
h.currentStatus.updateLock.Lock()
|
||||
h.currentStatus.Summary = healStoppedStatus
|
||||
h.currentStatus.FailureDetail = errHealStopSignalled.Error()
|
||||
h.currentStatus.updateLock.Unlock()
|
||||
|
||||
// drain traverse channel so the traversal
|
||||
// go-routine does not leak.
|
||||
go func() {
|
||||
// Eventually the traversal go-routine closes
|
||||
// the channel and returns, so this go-routine
|
||||
// itself will not leak.
|
||||
<-h.traverseAndHealDoneCh
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
// traverseAndHeal - traverses on-disk data and performs healing
|
||||
// according to settings. At each "safe" point it also checks if an
|
||||
// external quit signal has been received and quits if so. Since the
|
||||
// healing traversal may be mutating on-disk data when an external
|
||||
// quit signal is received, this routine cannot quit immediately and
|
||||
// has to wait until a safe point is reached, such as between scanning
|
||||
// two objects.
|
||||
func (h *healSequence) traverseAndHeal() {
|
||||
var err error
|
||||
checkErr := func(f func() error) {
|
||||
switch {
|
||||
case err != nil:
|
||||
return
|
||||
case h.isQuitting():
|
||||
err = errHealStopSignalled
|
||||
return
|
||||
}
|
||||
err = f()
|
||||
}
|
||||
|
||||
// Start with format healing
|
||||
checkErr(h.healDiskFormat)
|
||||
|
||||
// Heal buckets and objects
|
||||
checkErr(h.healBuckets)
|
||||
|
||||
if err != nil {
|
||||
h.traverseAndHealDoneCh <- err
|
||||
}
|
||||
|
||||
close(h.traverseAndHealDoneCh)
|
||||
}
|
||||
|
||||
// healDiskFormat - heals format.json, return value indicates if a
|
||||
// failure error occurred.
|
||||
func (h *healSequence) healDiskFormat() error {
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
// Acquire lock on format.json
|
||||
formatLock := globalNSMutex.NewNSLock(minioMetaBucket, formatConfigFile)
|
||||
if err := formatLock.GetLock(globalHealingTimeout); err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
}
|
||||
defer formatLock.Unlock()
|
||||
|
||||
// Create a new set of storage instances to heal format.json.
|
||||
bootstrapDisks, err := initStorageDisks(globalEndpoints)
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
}
|
||||
|
||||
// Wrap into retrying disks
|
||||
retryingDisks := initRetryableStorageDisks(bootstrapDisks,
|
||||
time.Millisecond, time.Millisecond*5,
|
||||
globalStorageHealthCheckInterval, globalStorageRetryThreshold)
|
||||
|
||||
// Heal format.json on available storage.
|
||||
hres, err := healFormatXL(retryingDisks, h.settings.DryRun)
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
}
|
||||
|
||||
// reload object layer global only if we healed some disk
|
||||
onlineBefore, onlineAfter := hres.GetOnlineCounts()
|
||||
numHealed := onlineAfter - onlineBefore
|
||||
if numHealed > 0 {
|
||||
// Instantiate new object layer with newly formatted
|
||||
// storage.
|
||||
newObjectAPI, err := newXLObjects(retryingDisks)
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
}
|
||||
|
||||
// Set object layer with newly formatted storage to
|
||||
// globalObjectAPI.
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = newObjectAPI
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Shutdown storage belonging to old object layer
|
||||
// instance.
|
||||
objectAPI.Shutdown()
|
||||
|
||||
// Inform peers to reinitialize storage with newly
|
||||
// formatted storage.
|
||||
reInitPeerDisks(globalAdminPeers)
|
||||
}
|
||||
|
||||
// Push format heal result
|
||||
return h.pushHealResultItem(hres)
|
||||
}
|
||||
|
||||
// healBuckets - check for all buckets heal or just particular bucket.
|
||||
func (h *healSequence) healBuckets() error {
|
||||
// 1. If a bucket was specified, heal only the bucket.
|
||||
if h.bucket != "" {
|
||||
return h.healBucket(h.bucket)
|
||||
}
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
buckets, err := objectAPI.ListBucketsHeal()
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
}
|
||||
|
||||
for _, bucket := range buckets {
|
||||
err = h.healBucket(bucket.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// healBucket - traverses and heals given bucket
|
||||
func (h *healSequence) healBucket(bucket string) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
results, err := objectAPI.HealBucket(bucket, h.settings.DryRun)
|
||||
// push any available results before checking for error
|
||||
for _, result := range results {
|
||||
if perr := h.pushHealResultItem(result); perr != nil {
|
||||
return perr
|
||||
}
|
||||
}
|
||||
// handle heal-bucket error
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !h.settings.Recursive {
|
||||
if h.objPrefix != "" {
|
||||
// Check if an object named as the objPrefix exists,
|
||||
// and if so heal it.
|
||||
_, err = objectAPI.GetObjectInfo(bucket, h.objPrefix)
|
||||
if err == nil {
|
||||
err = h.healObject(bucket, h.objPrefix)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
marker := ""
|
||||
isTruncated := true
|
||||
for isTruncated {
|
||||
objectInfos, err := objectAPI.ListObjectsHeal(bucket,
|
||||
h.objPrefix, marker, "", 1000)
|
||||
if err != nil {
|
||||
return errFnHealFromAPIErr(err)
|
||||
}
|
||||
|
||||
for _, o := range objectInfos.Objects {
|
||||
if err := h.healObject(o.Bucket, o.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
isTruncated = objectInfos.IsTruncated
|
||||
marker = objectInfos.NextMarker
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// healObject - heal the given object and record result
|
||||
func (h *healSequence) healObject(bucket, object string) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
hri, err := objectAPI.HealObject(bucket, object, h.settings.DryRun)
|
||||
if err != nil {
|
||||
hri.Detail = err.Error()
|
||||
}
|
||||
return h.pushHealResultItem(hri)
|
||||
}
|
||||
+30
-28
@@ -16,7 +16,15 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import router "github.com/gorilla/mux"
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
const (
|
||||
adminAPIPathPrefix = "/minio/admin"
|
||||
)
|
||||
|
||||
// adminAPIHandlers provides HTTP handlers for Minio admin API.
|
||||
type adminAPIHandlers struct {
|
||||
@@ -27,50 +35,44 @@ func registerAdminRouter(mux *router.Router) {
|
||||
|
||||
adminAPI := adminAPIHandlers{}
|
||||
// Admin router
|
||||
adminRouter := mux.NewRoute().PathPrefix("/").Subrouter()
|
||||
adminRouter := mux.NewRoute().PathPrefix(adminAPIPathPrefix).Subrouter()
|
||||
|
||||
// Version handler
|
||||
adminRouter.Methods(http.MethodGet).Path("/version").HandlerFunc(adminAPI.VersionHandler)
|
||||
|
||||
adminV1Router := adminRouter.PathPrefix("/v1").Subrouter()
|
||||
|
||||
/// Service operations
|
||||
|
||||
// Service status
|
||||
adminRouter.Methods("GET").Queries("service", "").Headers(minioAdminOpHeader, "status").HandlerFunc(adminAPI.ServiceStatusHandler)
|
||||
adminV1Router.Methods(http.MethodGet).Path("/service").HandlerFunc(adminAPI.ServiceStatusHandler)
|
||||
|
||||
// Service restart
|
||||
adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "restart").HandlerFunc(adminAPI.ServiceRestartHandler)
|
||||
// Service update credentials
|
||||
adminRouter.Methods("POST").Queries("service", "").Headers(minioAdminOpHeader, "set-credentials").HandlerFunc(adminAPI.ServiceCredentialsHandler)
|
||||
// Service restart and stop - TODO
|
||||
adminV1Router.Methods(http.MethodPost).Path("/service").HandlerFunc(adminAPI.ServiceStopNRestartHandler)
|
||||
|
||||
// Info operations
|
||||
adminRouter.Methods("GET").Queries("info", "").HandlerFunc(adminAPI.ServerInfoHandler)
|
||||
adminV1Router.Methods(http.MethodGet).Path("/info").HandlerFunc(adminAPI.ServerInfoHandler)
|
||||
|
||||
/// Lock operations
|
||||
|
||||
// List Locks
|
||||
adminRouter.Methods("GET").Queries("lock", "").Headers(minioAdminOpHeader, "list").HandlerFunc(adminAPI.ListLocksHandler)
|
||||
adminV1Router.Methods(http.MethodGet).Path("/locks").HandlerFunc(adminAPI.ListLocksHandler)
|
||||
// Clear locks
|
||||
adminRouter.Methods("POST").Queries("lock", "").Headers(minioAdminOpHeader, "clear").HandlerFunc(adminAPI.ClearLocksHandler)
|
||||
adminV1Router.Methods(http.MethodDelete).Path("/locks").HandlerFunc(adminAPI.ClearLocksHandler)
|
||||
|
||||
/// Heal operations
|
||||
|
||||
// List Objects needing heal.
|
||||
adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-objects").HandlerFunc(adminAPI.ListObjectsHealHandler)
|
||||
// List Uploads needing heal.
|
||||
adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-uploads").HandlerFunc(adminAPI.ListUploadsHealHandler)
|
||||
// List Buckets needing heal.
|
||||
adminRouter.Methods("GET").Queries("heal", "").Headers(minioAdminOpHeader, "list-buckets").HandlerFunc(adminAPI.ListBucketsHealHandler)
|
||||
|
||||
// Heal Buckets.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "bucket").HandlerFunc(adminAPI.HealBucketHandler)
|
||||
// Heal Objects.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "object").HandlerFunc(adminAPI.HealObjectHandler)
|
||||
// Heal Format.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "format").HandlerFunc(adminAPI.HealFormatHandler)
|
||||
// Heal Uploads.
|
||||
adminRouter.Methods("POST").Queries("heal", "").Headers(minioAdminOpHeader, "upload").HandlerFunc(adminAPI.HealUploadHandler)
|
||||
// Heal processing endpoint.
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/").HandlerFunc(adminAPI.HealHandler)
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}").HandlerFunc(adminAPI.HealHandler)
|
||||
adminV1Router.Methods(http.MethodPost).Path("/heal/{bucket}/{prefix:.*}").HandlerFunc(adminAPI.HealHandler)
|
||||
|
||||
/// Config operations
|
||||
|
||||
// Update credentials
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config/credential").HandlerFunc(adminAPI.UpdateCredentialsHandler)
|
||||
// Get config
|
||||
adminRouter.Methods("GET").Queries("config", "").Headers(minioAdminOpHeader, "get").HandlerFunc(adminAPI.GetConfigHandler)
|
||||
// Set Config
|
||||
adminRouter.Methods("PUT").Queries("config", "").Headers(minioAdminOpHeader, "set").HandlerFunc(adminAPI.SetConfigHandler)
|
||||
adminV1Router.Methods(http.MethodGet).Path("/config").HandlerFunc(adminAPI.GetConfigHandler)
|
||||
// Set config
|
||||
adminV1Router.Methods(http.MethodPut).Path("/config").HandlerFunc(adminAPI.SetConfigHandler)
|
||||
}
|
||||
|
||||
+42
-31
@@ -18,23 +18,22 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
// Admin service names
|
||||
serviceRestartRPC = "Admin.Restart"
|
||||
signalServiceRPC = "Admin.SignalService"
|
||||
listLocksRPC = "Admin.ListLocks"
|
||||
reInitDisksRPC = "Admin.ReInitDisks"
|
||||
serverInfoDataRPC = "Admin.ServerInfoData"
|
||||
@@ -56,7 +55,7 @@ type remoteAdminClient struct {
|
||||
// adminCmdRunner - abstracts local and remote execution of admin
|
||||
// commands like service stop and service restart.
|
||||
type adminCmdRunner interface {
|
||||
Restart() error
|
||||
SignalService(s serviceSignal) error
|
||||
ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error)
|
||||
ReInitDisks() error
|
||||
ServerInfoData() (ServerInfoData, error)
|
||||
@@ -65,10 +64,16 @@ type adminCmdRunner interface {
|
||||
CommitConfig(tmpFileName string) error
|
||||
}
|
||||
|
||||
// Restart - Sends a message over channel to the go-routine
|
||||
// responsible for restarting the process.
|
||||
func (lc localAdminClient) Restart() error {
|
||||
globalServiceSignalCh <- serviceRestart
|
||||
var errUnsupportedSignal = fmt.Errorf("unsupported signal: only restart and stop signals are supported")
|
||||
|
||||
// SignalService - sends a restart or stop signal to the local server
|
||||
func (lc localAdminClient) SignalService(s serviceSignal) error {
|
||||
switch s {
|
||||
case serviceRestart, serviceStop:
|
||||
globalServiceSignalCh <- s
|
||||
default:
|
||||
return errUnsupportedSignal
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -77,25 +82,31 @@ func (lc localAdminClient) ListLocks(bucket, prefix string, duration time.Durati
|
||||
return listLocksInfo(bucket, prefix, duration), nil
|
||||
}
|
||||
|
||||
// Restart - Sends restart command to remote server via RPC.
|
||||
func (rc remoteAdminClient) Restart() error {
|
||||
args := AuthRPCArgs{}
|
||||
reply := AuthRPCReply{}
|
||||
return rc.Call(serviceRestartRPC, &args, &reply)
|
||||
func (rc remoteAdminClient) SignalService(s serviceSignal) (err error) {
|
||||
switch s {
|
||||
case serviceRestart, serviceStop:
|
||||
reply := AuthRPCReply{}
|
||||
err = rc.Call(signalServiceRPC, &SignalServiceArgs{Sig: s},
|
||||
&reply)
|
||||
default:
|
||||
err = errUnsupportedSignal
|
||||
}
|
||||
return err
|
||||
|
||||
}
|
||||
|
||||
// ListLocks - Sends list locks command to remote server via RPC.
|
||||
func (rc remoteAdminClient) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
|
||||
listArgs := ListLocksQuery{
|
||||
bucket: bucket,
|
||||
prefix: prefix,
|
||||
duration: duration,
|
||||
Bucket: bucket,
|
||||
Prefix: prefix,
|
||||
Duration: duration,
|
||||
}
|
||||
var reply ListLocksReply
|
||||
if err := rc.Call(listLocksRPC, &listArgs, &reply); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return reply.volLocks, nil
|
||||
return reply.VolLocks, nil
|
||||
}
|
||||
|
||||
// ReInitDisks - There is nothing to do here, heal format REST API
|
||||
@@ -139,7 +150,7 @@ func (lc localAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
|
||||
Version: Version,
|
||||
CommitID: CommitID,
|
||||
SQSARN: arns,
|
||||
Region: serverConfig.GetRegion(),
|
||||
Region: globalServerConfig.GetRegion(),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
@@ -158,11 +169,11 @@ func (rc remoteAdminClient) ServerInfoData() (sid ServerInfoData, e error) {
|
||||
|
||||
// GetConfig - returns config.json of the local server.
|
||||
func (lc localAdminClient) GetConfig() ([]byte, error) {
|
||||
if serverConfig == nil {
|
||||
return nil, errors.New("config not present")
|
||||
if globalServerConfig == nil {
|
||||
return nil, fmt.Errorf("config not present")
|
||||
}
|
||||
|
||||
return json.Marshal(serverConfig)
|
||||
return json.Marshal(globalServerConfig)
|
||||
}
|
||||
|
||||
// GetConfig - returns config.json of the remote server.
|
||||
@@ -225,7 +236,7 @@ func (rc remoteAdminClient) CommitConfig(tmpFileName string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// adminPeer - represents an entity that implements Restart methods.
|
||||
// adminPeer - represents an entity that implements admin API RPCs.
|
||||
type adminPeer struct {
|
||||
addr string
|
||||
cmdRunner adminCmdRunner
|
||||
@@ -246,7 +257,7 @@ func makeAdminPeers(endpoints EndpointList) (adminPeerList adminPeers) {
|
||||
})
|
||||
|
||||
hostSet := set.CreateStringSet(globalMinioAddr)
|
||||
cred := serverConfig.GetCredential()
|
||||
cred := globalServerConfig.GetCredential()
|
||||
serviceEndpoint := path.Join(minioReservedBucketPath, adminPath)
|
||||
for _, host := range GetRemotePeers(endpoints) {
|
||||
if hostSet.Contains(host) {
|
||||
@@ -274,11 +285,11 @@ func initGlobalAdminPeers(endpoints EndpointList) {
|
||||
globalAdminPeers = makeAdminPeers(endpoints)
|
||||
}
|
||||
|
||||
// invokeServiceCmd - Invoke Restart command.
|
||||
// invokeServiceCmd - Invoke Restart/Stop command.
|
||||
func invokeServiceCmd(cp adminPeer, cmd serviceSignal) (err error) {
|
||||
switch cmd {
|
||||
case serviceRestart:
|
||||
err = cp.cmdRunner.Restart()
|
||||
case serviceRestart, serviceStop:
|
||||
err = cp.cmdRunner.SignalService(cmd)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -466,7 +477,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
|
||||
// Find the maximally occurring config among peers in a
|
||||
// distributed setup.
|
||||
|
||||
serverConfigs := make([]serverConfigV13, len(peers))
|
||||
serverConfigs := make([]serverConfig, len(peers))
|
||||
for i, configBytes := range configs {
|
||||
if errs[i] != nil {
|
||||
continue
|
||||
@@ -483,7 +494,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
|
||||
configJSON, err := getValidServerConfig(serverConfigs, errs)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find a valid server config")
|
||||
return nil, traceError(err)
|
||||
return nil, errors.Trace(err)
|
||||
}
|
||||
|
||||
// Return the config.json that was present quorum or more
|
||||
@@ -493,7 +504,7 @@ func getPeerConfig(peers adminPeers) ([]byte, error) {
|
||||
|
||||
// getValidServerConfig - finds the server config that is present in
|
||||
// quorum or more number of servers.
|
||||
func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv serverConfigV13, e error) {
|
||||
func getValidServerConfig(serverConfigs []serverConfig, errs []error) (scv serverConfig, e error) {
|
||||
// majority-based quorum
|
||||
quorum := len(serverConfigs)/2 + 1
|
||||
|
||||
@@ -536,7 +547,7 @@ func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv se
|
||||
// seen. See example above for
|
||||
// clarity.
|
||||
continue
|
||||
} else if j < i && reflect.DeepEqual(serverConfigs[i], serverConfigs[j]) {
|
||||
} else if j < i && serverConfigs[i].ConfigDiff(&serverConfigs[j]) == "" {
|
||||
// serverConfigs[i] is equal to
|
||||
// serverConfigs[j], update
|
||||
// serverConfigs[j]'s counter since it
|
||||
@@ -555,7 +566,7 @@ func getValidServerConfig(serverConfigs []serverConfigV13, errs []error) (scv se
|
||||
|
||||
// We find the maximally occurring server config and check if
|
||||
// there is quorum.
|
||||
var configJSON serverConfigV13
|
||||
var configJSON serverConfig
|
||||
maxOccurrence := 0
|
||||
for i, count := range configCounter {
|
||||
if maxOccurrence < count {
|
||||
|
||||
@@ -220,7 +220,7 @@ var (
|
||||
|
||||
// TestGetValidServerConfig - test for getValidServerConfig.
|
||||
func TestGetValidServerConfig(t *testing.T) {
|
||||
var c1, c2 serverConfigV13
|
||||
var c1, c2 serverConfig
|
||||
err := json.Unmarshal(config1, &c1)
|
||||
if err != nil {
|
||||
t.Fatalf("json unmarshal of %s failed: %v", string(config1), err)
|
||||
@@ -233,7 +233,7 @@ func TestGetValidServerConfig(t *testing.T) {
|
||||
|
||||
// Valid config.
|
||||
noErrs := []error{nil, nil, nil, nil}
|
||||
serverConfigs := []serverConfigV13{c1, c2, c1, c1}
|
||||
serverConfigs := []serverConfig{c1, c2, c1, c1}
|
||||
validConfig, err := getValidServerConfig(serverConfigs, noErrs)
|
||||
if err != nil {
|
||||
t.Errorf("Expected a valid config but received %v instead", err)
|
||||
@@ -244,7 +244,7 @@ func TestGetValidServerConfig(t *testing.T) {
|
||||
}
|
||||
|
||||
// Invalid config - no quorum.
|
||||
serverConfigs = []serverConfigV13{c1, c2, c2, c1}
|
||||
serverConfigs = []serverConfig{c1, c2, c2, c1}
|
||||
_, err = getValidServerConfig(serverConfigs, noErrs)
|
||||
if err != errXLWriteQuorum {
|
||||
t.Errorf("Expected to fail due to lack of quorum but received %v", err)
|
||||
@@ -252,7 +252,7 @@ func TestGetValidServerConfig(t *testing.T) {
|
||||
|
||||
// All errors
|
||||
allErrs := []error{errDiskNotFound, errDiskNotFound, errDiskNotFound, errDiskNotFound}
|
||||
serverConfigs = []serverConfigV13{{}, {}, {}, {}}
|
||||
serverConfigs = []serverConfig{{}, {}, {}, {}}
|
||||
_, err = getValidServerConfig(serverConfigs, allErrs)
|
||||
if err != errXLWriteQuorum {
|
||||
t.Errorf("Expected to fail due to lack of quorum but received %v", err)
|
||||
|
||||
+27
-17
@@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
@@ -26,11 +25,12 @@ import (
|
||||
"time"
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
const adminPath = "/admin"
|
||||
|
||||
var errUnsupportedBackend = errors.New("not supported for non erasure-code backend")
|
||||
var errUnsupportedBackend = fmt.Errorf("not supported for non erasure-code backend")
|
||||
|
||||
// adminCmd - exports RPC methods for service status, stop and
|
||||
// restart commands.
|
||||
@@ -38,18 +38,24 @@ type adminCmd struct {
|
||||
AuthRPCServer
|
||||
}
|
||||
|
||||
// SignalServiceArgs - provides the signal argument to SignalService RPC
|
||||
type SignalServiceArgs struct {
|
||||
AuthRPCArgs
|
||||
Sig serviceSignal
|
||||
}
|
||||
|
||||
// ListLocksQuery - wraps ListLocks API's query values to send over RPC.
|
||||
type ListLocksQuery struct {
|
||||
AuthRPCArgs
|
||||
bucket string
|
||||
prefix string
|
||||
duration time.Duration
|
||||
Bucket string
|
||||
Prefix string
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// ListLocksReply - wraps ListLocks response over RPC.
|
||||
type ListLocksReply struct {
|
||||
AuthRPCReply
|
||||
volLocks []VolumeLockInfo
|
||||
VolLocks []VolumeLockInfo
|
||||
}
|
||||
|
||||
// ServerInfoDataReply - wraps the server info response over RPC.
|
||||
@@ -64,13 +70,13 @@ type ConfigReply struct {
|
||||
Config []byte // json-marshalled bytes of serverConfigV13
|
||||
}
|
||||
|
||||
// Restart - Restart this instance of minio server.
|
||||
func (s *adminCmd) Restart(args *AuthRPCArgs, reply *AuthRPCReply) error {
|
||||
// SignalService - Send a restart or stop signal to the service
|
||||
func (s *adminCmd) SignalService(args *SignalServiceArgs, reply *AuthRPCReply) error {
|
||||
if err := args.IsAuthenticated(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
globalServiceSignalCh <- serviceRestart
|
||||
globalServiceSignalCh <- args.Sig
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -79,8 +85,8 @@ func (s *adminCmd) ListLocks(query *ListLocksQuery, reply *ListLocksReply) error
|
||||
if err := query.IsAuthenticated(); err != nil {
|
||||
return err
|
||||
}
|
||||
volLocks := listLocksInfo(query.bucket, query.prefix, query.duration)
|
||||
*reply = ListLocksReply{volLocks: volLocks}
|
||||
volLocks := listLocksInfo(query.Bucket, query.Prefix, query.Duration)
|
||||
*reply = ListLocksReply{VolLocks: volLocks}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -104,8 +110,12 @@ func (s *adminCmd) ReInitDisks(args *AuthRPCArgs, reply *AuthRPCReply) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// Wrap into retrying disks
|
||||
retryingDisks := initRetryableStorageDisks(bootstrapDisks,
|
||||
time.Millisecond, time.Millisecond*5, globalStorageHealthCheckInterval, globalStorageRetryThreshold)
|
||||
|
||||
// Initialize new object layer with newly formatted disks.
|
||||
newObjectAPI, err := newXLObjects(bootstrapDisks)
|
||||
newObjectAPI, err := newXLObjects(retryingDisks)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -148,7 +158,7 @@ func (s *adminCmd) ServerInfoData(args *AuthRPCArgs, reply *ServerInfoDataReply)
|
||||
Uptime: UTCNow().Sub(globalBootTime),
|
||||
Version: Version,
|
||||
CommitID: CommitID,
|
||||
Region: serverConfig.GetRegion(),
|
||||
Region: globalServerConfig.GetRegion(),
|
||||
SQSARN: arns,
|
||||
},
|
||||
StorageInfo: storageInfo,
|
||||
@@ -165,11 +175,11 @@ func (s *adminCmd) GetConfig(args *AuthRPCArgs, reply *ConfigReply) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if serverConfig == nil {
|
||||
return errors.New("config not present")
|
||||
if globalServerConfig == nil {
|
||||
return fmt.Errorf("config not present")
|
||||
}
|
||||
|
||||
jsonBytes, err := json.Marshal(serverConfig)
|
||||
jsonBytes, err := json.Marshal(globalServerConfig)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -238,7 +248,7 @@ func registerAdminRPCRouter(mux *router.Router) error {
|
||||
adminRPCServer := newRPCServer()
|
||||
err := adminRPCServer.RegisterName("Admin", adminRPCHandler)
|
||||
if err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
adminRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
|
||||
adminRouter.Path(adminPath).Handler(adminRPCServer)
|
||||
|
||||
@@ -23,8 +23,8 @@ import (
|
||||
)
|
||||
|
||||
func testAdminCmd(cmd cmdType, t *testing.T) {
|
||||
// reset globals.
|
||||
// this is to make sure that the tests are not affected by modified globals.
|
||||
// reset globals. this is to make sure that the tests are not
|
||||
// affected by modified globals.
|
||||
resetTestGlobals()
|
||||
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
@@ -33,7 +33,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
creds := serverConfig.GetCredential()
|
||||
creds := globalServerConfig.GetCredential()
|
||||
token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -42,7 +42,7 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
|
||||
adminServer := adminCmd{}
|
||||
args := LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
err = adminServer.Login(&args, &LoginRPCReply{})
|
||||
@@ -55,12 +55,23 @@ func testAdminCmd(cmd cmdType, t *testing.T) {
|
||||
<-globalServiceSignalCh
|
||||
}()
|
||||
|
||||
ga := AuthRPCArgs{AuthToken: token}
|
||||
sa := SignalServiceArgs{
|
||||
AuthRPCArgs: AuthRPCArgs{AuthToken: token, Version: globalRPCAPIVersion},
|
||||
Sig: cmd.toServiceSignal(),
|
||||
}
|
||||
|
||||
genReply := AuthRPCReply{}
|
||||
switch cmd {
|
||||
case restartCmd:
|
||||
if err = adminServer.Restart(&ga, &genReply); err != nil {
|
||||
t.Errorf("restartCmd: Expected: <nil>, got: %v", err)
|
||||
case restartCmd, stopCmd:
|
||||
if err = adminServer.SignalService(&sa, &genReply); err != nil {
|
||||
t.Errorf("restartCmd/stopCmd: Expected: <nil>, got: %v",
|
||||
err)
|
||||
}
|
||||
default:
|
||||
err = adminServer.SignalService(&sa, &genReply)
|
||||
if err != nil && err.Error() != errUnsupportedSignal.Error() {
|
||||
t.Errorf("invalidSignal %s: unexpected error got: %v",
|
||||
cmd, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -70,6 +81,16 @@ func TestAdminRestart(t *testing.T) {
|
||||
testAdminCmd(restartCmd, t)
|
||||
}
|
||||
|
||||
// TestAdminStop - test for Admin.Stop RPC service.
|
||||
func TestAdminStop(t *testing.T) {
|
||||
testAdminCmd(stopCmd, t)
|
||||
}
|
||||
|
||||
// TestAdminStatus - test for Admin.Status RPC service (error case)
|
||||
func TestAdminStatus(t *testing.T) {
|
||||
testAdminCmd(statusCmd, t)
|
||||
}
|
||||
|
||||
// TestReInitDisks - test for Admin.ReInitDisks RPC service.
|
||||
func TestReInitDisks(t *testing.T) {
|
||||
// Reset global variables to start afresh.
|
||||
@@ -95,7 +116,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
globalIsXL = true
|
||||
adminServer := adminCmd{}
|
||||
|
||||
creds := serverConfig.GetCredential()
|
||||
creds := globalServerConfig.GetCredential()
|
||||
token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -103,7 +124,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
|
||||
args := LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
err = adminServer.Login(&args, &LoginRPCReply{})
|
||||
@@ -113,6 +134,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
|
||||
authArgs := AuthRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: globalRPCAPIVersion,
|
||||
}
|
||||
authReply := AuthRPCReply{}
|
||||
|
||||
@@ -130,7 +152,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
fsAdminServer := adminCmd{}
|
||||
fsArgs := LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
fsReply := LoginRPCReply{}
|
||||
@@ -141,6 +163,7 @@ func TestReInitDisks(t *testing.T) {
|
||||
|
||||
authArgs = AuthRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: globalRPCAPIVersion,
|
||||
}
|
||||
authReply = AuthRPCReply{}
|
||||
// Attempt ReInitDisks service on a FS backend.
|
||||
@@ -163,7 +186,7 @@ func TestGetConfig(t *testing.T) {
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
adminServer := adminCmd{}
|
||||
creds := serverConfig.GetCredential()
|
||||
creds := globalServerConfig.GetCredential()
|
||||
|
||||
token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
@@ -172,7 +195,7 @@ func TestGetConfig(t *testing.T) {
|
||||
|
||||
args := LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
reply := LoginRPCReply{}
|
||||
@@ -183,6 +206,7 @@ func TestGetConfig(t *testing.T) {
|
||||
|
||||
authArgs := AuthRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: globalRPCAPIVersion,
|
||||
}
|
||||
|
||||
configReply := ConfigReply{}
|
||||
@@ -212,14 +236,14 @@ func TestWriteAndCommitConfig(t *testing.T) {
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
adminServer := adminCmd{}
|
||||
creds := serverConfig.GetCredential()
|
||||
creds := globalServerConfig.GetCredential()
|
||||
token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
args := LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
reply := LoginRPCReply{}
|
||||
@@ -234,6 +258,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
|
||||
wArgs := WriteConfigArgs{
|
||||
AuthRPCArgs: AuthRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: globalRPCAPIVersion,
|
||||
},
|
||||
TmpFileName: tmpFileName,
|
||||
Buf: buf,
|
||||
@@ -251,6 +276,7 @@ func TestWriteAndCommitConfig(t *testing.T) {
|
||||
cArgs := CommitConfigArgs{
|
||||
AuthRPCArgs: AuthRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: globalRPCAPIVersion,
|
||||
},
|
||||
FileName: tmpFileName,
|
||||
}
|
||||
|
||||
+88
-4
@@ -18,9 +18,11 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
@@ -39,8 +41,8 @@ type APIErrorResponse struct {
|
||||
Key string
|
||||
BucketName string
|
||||
Resource string
|
||||
RequestID string `xml:"RequestId"`
|
||||
HostID string `xml:"HostId"`
|
||||
RequestID string `xml:"RequestId" json:"RequestId"`
|
||||
HostID string `xml:"HostId" json:"HostId"`
|
||||
}
|
||||
|
||||
// APIErrorCode type of error status.
|
||||
@@ -121,10 +123,11 @@ const (
|
||||
ErrMetadataTooLarge
|
||||
ErrUnsupportedMetadata
|
||||
ErrMaximumExpires
|
||||
ErrSlowDown
|
||||
ErrInvalidPrefixMarker
|
||||
// Add new error codes here.
|
||||
|
||||
// Server-Side-Encryption (with Customer provided key) related API errors.
|
||||
|
||||
ErrInsecureSSECustomerRequest
|
||||
ErrSSEEncryptedObject
|
||||
ErrInvalidEncryptionParameters
|
||||
@@ -155,6 +158,7 @@ const (
|
||||
ErrReadQuorum
|
||||
ErrWriteQuorum
|
||||
ErrStorageFull
|
||||
ErrRequestBodyParse
|
||||
ErrObjectExistsAsDirectory
|
||||
ErrPolicyNesting
|
||||
ErrInvalidObjectName
|
||||
@@ -163,16 +167,29 @@ const (
|
||||
ErrOperationTimedOut
|
||||
ErrPartsSizeUnequal
|
||||
ErrInvalidRequest
|
||||
|
||||
// Minio storage class error codes
|
||||
ErrInvalidStorageClass
|
||||
|
||||
// Add new extended error codes here.
|
||||
// Please open a https://github.com/minio/minio/issues before adding
|
||||
// new error codes here.
|
||||
|
||||
ErrMalformedJSON
|
||||
ErrAdminInvalidAccessKey
|
||||
ErrAdminInvalidSecretKey
|
||||
ErrAdminConfigNoQuorum
|
||||
ErrAdminConfigTooLarge
|
||||
ErrAdminConfigBadJSON
|
||||
ErrAdminCredentialsMismatch
|
||||
ErrInsecureClientRequest
|
||||
ErrObjectTampered
|
||||
ErrHealNotImplemented
|
||||
ErrHealNoSuchProcess
|
||||
ErrHealInvalidClientToken
|
||||
ErrHealMissingBucket
|
||||
ErrHealAlreadyRunning
|
||||
ErrHealOverlappingPaths
|
||||
)
|
||||
|
||||
// error code to APIError structure, these fields carry respective
|
||||
@@ -193,6 +210,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Unknown metadata directive.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidStorageClass: {
|
||||
Code: "InvalidStorageClass",
|
||||
Description: "Invalid storage class.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInvalidRequestBody: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Body shouldn't be set for this request.",
|
||||
@@ -505,6 +527,17 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Request is not valid yet",
|
||||
HTTPStatusCode: http.StatusForbidden,
|
||||
},
|
||||
ErrSlowDown: {
|
||||
Code: "SlowDown",
|
||||
Description: "Please reduce your request",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrInvalidPrefixMarker: {
|
||||
Code: "InvalidPrefixMarker",
|
||||
Description: "Invalid marker prefix combination",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
|
||||
// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
|
||||
ErrUnsignedHeaders: {
|
||||
Code: "AccessDenied",
|
||||
@@ -647,6 +680,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Storage backend has reached its minimum free disk threshold. Please delete a few objects to proceed.",
|
||||
HTTPStatusCode: http.StatusInternalServerError,
|
||||
},
|
||||
ErrRequestBodyParse: {
|
||||
Code: "XMinioRequestBodyParse",
|
||||
Description: "The request body failed to parse.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrObjectExistsAsDirectory: {
|
||||
Code: "XMinioObjectExistsAsDirectory",
|
||||
Description: "Object name already exists as a directory.",
|
||||
@@ -682,6 +720,11 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Server not initialized, please try again.",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrMalformedJSON: {
|
||||
Code: "XMinioMalformedJSON",
|
||||
Description: "The JSON you provided was not well-formed or did not validate against our published format.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminInvalidAccessKey: {
|
||||
Code: "XMinioAdminInvalidAccessKey",
|
||||
Description: "The access key is invalid.",
|
||||
@@ -697,6 +740,17 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Configuration update failed because server quorum was not met",
|
||||
HTTPStatusCode: http.StatusServiceUnavailable,
|
||||
},
|
||||
ErrAdminConfigTooLarge: {
|
||||
Code: "XMinioAdminConfigTooLarge",
|
||||
Description: fmt.Sprintf("Configuration data provided exceeds the allowed maximum of %d bytes",
|
||||
maxConfigJSONSize),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminConfigBadJSON: {
|
||||
Code: "XMinioAdminConfigBadJSON",
|
||||
Description: "JSON configuration provided has objects with duplicate keys",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminCredentialsMismatch: {
|
||||
Code: "XMinioAdminCredentialsMismatch",
|
||||
Description: "Credentials in config mismatch with server environment variables",
|
||||
@@ -740,6 +794,36 @@ var errorCodeResponse = map[APIErrorCode]APIError{
|
||||
Description: "Invalid Request",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrHealNotImplemented: {
|
||||
Code: "XMinioHealNotImplemented",
|
||||
Description: "This server does not implement heal functionality.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrHealNoSuchProcess: {
|
||||
Code: "XMinioHealNoSuchProcess",
|
||||
Description: "No such heal process is running on the server",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrHealInvalidClientToken: {
|
||||
Code: "XMinioHealInvalidClientToken",
|
||||
Description: "Client token mismatch",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrHealMissingBucket: {
|
||||
Code: "XMinioHealMissingBucket",
|
||||
Description: "A heal start request with a non-empty object-prefix parameter requires a bucket to be specified.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrHealAlreadyRunning: {
|
||||
Code: "XMinioHealAlreadyRunning",
|
||||
Description: "",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrHealOverlappingPaths: {
|
||||
Code: "XMinioHealOverlappingPaths",
|
||||
Description: "",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
|
||||
// Add your error structure here.
|
||||
}
|
||||
@@ -752,7 +836,7 @@ func toAPIErrorCode(err error) (apiErr APIErrorCode) {
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
err = errorCause(err)
|
||||
err = errors.Cause(err)
|
||||
// Verify if the underlying error is signature mismatch.
|
||||
switch err {
|
||||
case errSignatureMismatch:
|
||||
|
||||
+10
-1
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -38,7 +39,7 @@ func setCommonHeaders(w http.ResponseWriter) {
|
||||
w.Header().Set("Server", globalServerUserAgent)
|
||||
// Set `x-amz-bucket-region` only if region is set on the server
|
||||
// by default minio uses an empty region.
|
||||
if region := serverConfig.GetRegion(); region != "" {
|
||||
if region := globalServerConfig.GetRegion(); region != "" {
|
||||
w.Header().Set("X-Amz-Bucket-Region", region)
|
||||
}
|
||||
w.Header().Set("Accept-Ranges", "bytes")
|
||||
@@ -53,6 +54,14 @@ func encodeResponse(response interface{}) []byte {
|
||||
return bytesBuffer.Bytes()
|
||||
}
|
||||
|
||||
// Encodes the response headers into JSON format.
|
||||
func encodeResponseJSON(response interface{}) []byte {
|
||||
var bytesBuffer bytes.Buffer
|
||||
e := json.NewEncoder(&bytesBuffer)
|
||||
e.Encode(response)
|
||||
return bytesBuffer.Bytes()
|
||||
}
|
||||
|
||||
// Write object header
|
||||
func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, contentRange *httpRange) {
|
||||
// set common headers
|
||||
|
||||
+43
-16
@@ -166,13 +166,12 @@ type ListBucketsResponse struct {
|
||||
|
||||
// Upload container for in progress multipart upload
|
||||
type Upload struct {
|
||||
Key string
|
||||
UploadID string `xml:"UploadId"`
|
||||
Initiator Initiator
|
||||
Owner Owner
|
||||
StorageClass string
|
||||
Initiated string
|
||||
HealUploadInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"`
|
||||
Key string
|
||||
UploadID string `xml:"UploadId"`
|
||||
Initiator Initiator
|
||||
Owner Owner
|
||||
StorageClass string
|
||||
Initiated string
|
||||
}
|
||||
|
||||
// CommonPrefix container for prefix response in ListObjectsResponse
|
||||
@@ -182,9 +181,8 @@ type CommonPrefix struct {
|
||||
|
||||
// Bucket container for bucket metadata
|
||||
type Bucket struct {
|
||||
Name string
|
||||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
HealBucketInfo *HealBucketInfo `xml:"HealBucketInfo,omitempty"`
|
||||
Name string
|
||||
CreationDate string // time string of format "2006-01-02T15:04:05.000Z"
|
||||
}
|
||||
|
||||
// Object container for object metadata
|
||||
@@ -198,8 +196,7 @@ type Object struct {
|
||||
Owner Owner
|
||||
|
||||
// The class of storage used to store the object.
|
||||
StorageClass string
|
||||
HealObjectInfo *HealObjectInfo `xml:"HealObjectInfo,omitempty"`
|
||||
StorageClass string
|
||||
}
|
||||
|
||||
// CopyObjectResponse container returns ETag and LastModified of the successfully copied object
|
||||
@@ -308,7 +305,6 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
||||
var listbucket = Bucket{}
|
||||
listbucket.Name = bucket.Name
|
||||
listbucket.CreationDate = bucket.Created.UTC().Format(timeFormatAMZLong)
|
||||
listbucket.HealBucketInfo = bucket.HealBucketInfo
|
||||
listbuckets = append(listbuckets, listbucket)
|
||||
}
|
||||
|
||||
@@ -339,8 +335,6 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter string, max
|
||||
content.Size = object.Size
|
||||
content.StorageClass = globalMinioDefaultStorageClass
|
||||
content.Owner = owner
|
||||
// object.HealObjectInfo is non-empty only when resp is constructed in ListObjectsHeal.
|
||||
content.HealObjectInfo = object.HealObjectInfo
|
||||
contents = append(contents, content)
|
||||
}
|
||||
// TODO - support EncodingType in xml decoding
|
||||
@@ -498,7 +492,6 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
|
||||
newUpload.UploadID = upload.UploadID
|
||||
newUpload.Key = upload.Object
|
||||
newUpload.Initiated = upload.Initiated.UTC().Format(timeFormatAMZLong)
|
||||
newUpload.HealUploadInfo = upload.HealUploadInfo
|
||||
listMultipartUploadsResponse.Uploads[index] = newUpload
|
||||
}
|
||||
return listMultipartUploadsResponse
|
||||
@@ -567,6 +560,12 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
|
||||
|
||||
// writeErrorRespone writes error headers
|
||||
func writeErrorResponse(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
|
||||
switch errorCode {
|
||||
case ErrSlowDown, ErrServerNotInitialized, ErrReadQuorum, ErrWriteQuorum:
|
||||
// Set retry-after header to indicate user-agents to retry request after 120secs.
|
||||
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
|
||||
w.Header().Set("Retry-After", "120")
|
||||
}
|
||||
apiError := getAPIError(errorCode)
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
|
||||
@@ -578,3 +577,31 @@ func writeErrorResponseHeadersOnly(w http.ResponseWriter, errorCode APIErrorCode
|
||||
apiError := getAPIError(errorCode)
|
||||
writeResponse(w, apiError.HTTPStatusCode, nil, mimeNone)
|
||||
}
|
||||
|
||||
// writeErrorResponseJSON - writes error response in JSON format;
|
||||
// useful for admin APIs.
|
||||
func writeErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode, reqURL *url.URL) {
|
||||
apiError := getAPIError(errorCode)
|
||||
// Generate error response.
|
||||
errorResponse := getAPIErrorResponse(apiError, reqURL.Path)
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||
}
|
||||
|
||||
// writeCustomErrorResponseJSON - similar to writeErrorResponseJSON,
|
||||
// but accepts the error message directly (this allows messages to be
|
||||
// dynamically generated.)
|
||||
func writeCustomErrorResponseJSON(w http.ResponseWriter, errorCode APIErrorCode,
|
||||
errBody string, reqURL *url.URL) {
|
||||
|
||||
apiError := getAPIError(errorCode)
|
||||
errorResponse := APIErrorResponse{
|
||||
Code: apiError.Code,
|
||||
Message: errBody,
|
||||
Resource: reqURL.Path,
|
||||
RequestID: "3L137",
|
||||
HostID: "3L137",
|
||||
}
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
writeResponse(w, apiError.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||
}
|
||||
|
||||
+17
-7
@@ -18,16 +18,12 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Verify if the request http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD"
|
||||
func isRequestUnsignedPayload(r *http.Request) bool {
|
||||
return r.Header.Get("x-amz-content-sha256") == unsignedPayload
|
||||
}
|
||||
|
||||
// Verify if request has JWT.
|
||||
func isRequestJWT(r *http.Request) bool {
|
||||
return strings.HasPrefix(r.Header.Get("Authorization"), jwtAlgorithm)
|
||||
@@ -58,13 +54,14 @@ func isRequestPresignedSignatureV2(r *http.Request) bool {
|
||||
|
||||
// Verify if request has AWS Post policy Signature Version '4'.
|
||||
func isRequestPostPolicySignatureV4(r *http.Request) bool {
|
||||
return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") && r.Method == httpPOST
|
||||
return strings.Contains(r.Header.Get("Content-Type"), "multipart/form-data") &&
|
||||
r.Method == http.MethodPost
|
||||
}
|
||||
|
||||
// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
|
||||
func isRequestSignStreamingV4(r *http.Request) bool {
|
||||
return r.Header.Get("x-amz-content-sha256") == streamingContentSHA256 &&
|
||||
r.Method == httpPUT
|
||||
r.Method == http.MethodPut
|
||||
}
|
||||
|
||||
// Authorization type.
|
||||
@@ -105,6 +102,19 @@ func getRequestAuthType(r *http.Request) authType {
|
||||
return authTypeUnknown
|
||||
}
|
||||
|
||||
// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
|
||||
// It does not accept presigned or JWT or anonymous requests.
|
||||
func checkAdminRequestAuthType(r *http.Request, region string) APIErrorCode {
|
||||
s3Err := ErrAccessDenied
|
||||
if getRequestAuthType(r) == authTypeSigned { // we only support V4 (no presign)
|
||||
s3Err = isReqAuthenticated(r, region)
|
||||
}
|
||||
if s3Err != ErrNone {
|
||||
errorIf(errors.New(getAPIError(s3Err).Description), "%s", dumpRequest(r))
|
||||
}
|
||||
return s3Err
|
||||
}
|
||||
|
||||
func checkRequestAuthType(r *http.Request, bucket, policyAction, region string) APIErrorCode {
|
||||
reqAuthType := getRequestAuthType(r)
|
||||
|
||||
|
||||
+67
-38
@@ -23,6 +23,7 @@ import (
|
||||
"net/url"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
)
|
||||
@@ -190,40 +191,6 @@ func TestS3SupportedAuthType(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRequestUnsignedPayload - Test validates the Unsigned payload detection logic.
|
||||
func TestIsRequestUnsignedPayload(t *testing.T) {
|
||||
testCases := []struct {
|
||||
inputAmzContentHeader string
|
||||
expectedResult bool
|
||||
}{
|
||||
// Test case - 1.
|
||||
// Test case with "X-Amz-Content-Sha256" header set to empty value.
|
||||
{"", false},
|
||||
// Test case - 2.
|
||||
// Test case with "X-Amz-Content-Sha256" header set to "UNSIGNED-PAYLOAD"
|
||||
// The payload is flagged as unsigned When "X-Amz-Content-Sha256" header is set to "UNSIGNED-PAYLOAD".
|
||||
{unsignedPayload, true},
|
||||
// Test case - 3.
|
||||
// set to a random value.
|
||||
{"abcd", false},
|
||||
}
|
||||
|
||||
// creating an input HTTP request.
|
||||
// Only the headers are relevant for this particular test.
|
||||
inputReq, err := http.NewRequest("GET", "http://example.com", nil)
|
||||
if err != nil {
|
||||
t.Fatalf("Error initializing input HTTP request: %v", err)
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
inputReq.Header.Set("X-Amz-Content-Sha256", testCase.inputAmzContentHeader)
|
||||
actualResult := isRequestUnsignedPayload(inputReq)
|
||||
if testCase.expectedResult != actualResult {
|
||||
t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsRequestPresignedSignatureV2(t *testing.T) {
|
||||
testCases := []struct {
|
||||
inputQueryKey string
|
||||
@@ -304,17 +271,50 @@ func mustNewRequest(method string, urlStr string, contentLength int64, body io.R
|
||||
// is signed with AWS Signature V4, fails if not able to do so.
|
||||
func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
|
||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||
cred := serverConfig.GetCredential()
|
||||
cred := globalServerConfig.GetCredential()
|
||||
if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
|
||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// This is similar to mustNewRequest but additionally the request
|
||||
// is signed with AWS Signature V2, fails if not able to do so.
|
||||
func mustNewSignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
|
||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||
cred := globalServerConfig.GetCredential()
|
||||
if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil {
|
||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// This is similar to mustNewRequest but additionally the request
|
||||
// is presigned with AWS Signature V2, fails if not able to do so.
|
||||
func mustNewPresignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
|
||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||
cred := globalServerConfig.GetCredential()
|
||||
if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
|
||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
// This is similar to mustNewRequest but additionally the request
|
||||
// is presigned with AWS Signature V4, fails if not able to do so.
|
||||
func mustNewPresignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
|
||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||
cred := globalServerConfig.GetCredential()
|
||||
if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil {
|
||||
t.Fatalf("Unable to inititalized new signed http request %s", err)
|
||||
}
|
||||
return req
|
||||
}
|
||||
|
||||
func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request {
|
||||
req := mustNewRequest(method, urlStr, contentLength, body, t)
|
||||
req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==")
|
||||
cred := serverConfig.GetCredential()
|
||||
cred := globalServerConfig.GetCredential()
|
||||
if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil {
|
||||
t.Fatalf("Unable to initialized new signed http request %s", err)
|
||||
}
|
||||
@@ -334,7 +334,7 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
t.Fatalf("unable create credential, %s", err)
|
||||
}
|
||||
|
||||
serverConfig.SetCredential(creds)
|
||||
globalServerConfig.SetCredential(creds)
|
||||
|
||||
// List of test cases for validating http request authentication.
|
||||
testCases := []struct {
|
||||
@@ -353,8 +353,37 @@ func TestIsReqAuthenticated(t *testing.T) {
|
||||
|
||||
// Validates all testcases.
|
||||
for _, testCase := range testCases {
|
||||
if s3Error := isReqAuthenticated(testCase.req, serverConfig.GetRegion()); s3Error != testCase.s3Error {
|
||||
if s3Error := isReqAuthenticated(testCase.req, globalServerConfig.GetRegion()); s3Error != testCase.s3Error {
|
||||
t.Fatalf("Unexpected s3error returned wanted %d, got %d", testCase.s3Error, s3Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
path, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("unable initialize config file, %s", err)
|
||||
}
|
||||
defer os.RemoveAll(path)
|
||||
|
||||
creds, err := auth.CreateCredentials("myuser", "mypassword")
|
||||
if err != nil {
|
||||
t.Fatalf("unable create credential, %s", err)
|
||||
}
|
||||
|
||||
globalServerConfig.SetCredential(creds)
|
||||
testCases := []struct {
|
||||
Request *http.Request
|
||||
ErrCode APIErrorCode
|
||||
}{
|
||||
{Request: mustNewRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
{Request: mustNewSignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrNone},
|
||||
{Request: mustNewSignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
{Request: mustNewPresignedV2Request("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
{Request: mustNewPresignedRequest("GET", "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
if s3Error := checkAdminRequestAuthType(testCase.Request, globalServerConfig.GetRegion()); s3Error != testCase.ErrCode {
|
||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+139
-13
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -17,7 +17,16 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/rpc"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
@@ -52,10 +61,11 @@ type authConfig struct {
|
||||
|
||||
// AuthRPCClient is a authenticated RPC client which does authentication before doing Call().
|
||||
type AuthRPCClient struct {
|
||||
sync.RWMutex // Mutex to lock this object.
|
||||
rpcClient *RPCClient // Reconnectable RPC client to make any RPC call.
|
||||
config authConfig // Authentication configuration information.
|
||||
authToken string // Authentication token.
|
||||
sync.RWMutex // Mutex to lock this object.
|
||||
rpcClient *rpc.Client // RPC client to make any RPC call.
|
||||
config authConfig // Authentication configuration information.
|
||||
authToken string // Authentication token.
|
||||
version semVersion // RPC version.
|
||||
}
|
||||
|
||||
// newAuthRPCClient - returns a JWT based authenticated (go) rpc client, which does automatic reconnect.
|
||||
@@ -73,8 +83,8 @@ func newAuthRPCClient(config authConfig) *AuthRPCClient {
|
||||
}
|
||||
|
||||
return &AuthRPCClient{
|
||||
rpcClient: newRPCClient(config.serverAddr, config.serviceEndpoint, config.secureConn),
|
||||
config: config,
|
||||
config: config,
|
||||
version: globalRPCAPIVersion,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -99,38 +109,60 @@ func (authClient *AuthRPCClient) Login() (err error) {
|
||||
|
||||
// Attempt to login if not logged in already.
|
||||
if authClient.authToken == "" {
|
||||
authClient.authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
|
||||
var authToken string
|
||||
authToken, err = authenticateNode(authClient.config.accessKey, authClient.config.secretKey)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Login to authenticate your token.
|
||||
var (
|
||||
loginMethod = authClient.config.serviceName + loginMethodName
|
||||
loginArgs = LoginRPCArgs{
|
||||
AuthToken: authClient.authToken,
|
||||
Version: Version,
|
||||
AuthToken: authToken,
|
||||
Version: globalRPCAPIVersion,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
)
|
||||
if err = authClient.rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
|
||||
|
||||
// Re-dial after we have disconnected or if its a fresh run.
|
||||
var rpcClient *rpc.Client
|
||||
rpcClient, err = rpcDial(authClient.config.serverAddr, authClient.config.serviceEndpoint, authClient.config.secureConn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = rpcClient.Call(loginMethod, &loginArgs, &LoginRPCReply{}); err != nil {
|
||||
// gob doesn't provide any typed errors for us to reflect
|
||||
// upon, this is the only way to return proper error.
|
||||
if strings.Contains(err.Error(), "gob: wrong type") {
|
||||
return errRPCAPIVersionUnsupported
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Initialize rpc client and auth token after a successful login.
|
||||
authClient.authToken = authToken
|
||||
authClient.rpcClient = rpcClient
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// call makes a RPC call after logs into the server.
|
||||
func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
|
||||
SetAuthToken(authToken string)
|
||||
SetRPCAPIVersion(version semVersion)
|
||||
}, reply interface{}) (err error) {
|
||||
if err = authClient.Login(); err != nil {
|
||||
return err
|
||||
} // On successful login, execute RPC call.
|
||||
|
||||
authClient.RLock()
|
||||
// Set token before the rpc call.
|
||||
authClient.RLock()
|
||||
defer authClient.RUnlock()
|
||||
args.SetAuthToken(authClient.authToken)
|
||||
authClient.RUnlock()
|
||||
args.SetRPCAPIVersion(authClient.version)
|
||||
|
||||
// Do an RPC call.
|
||||
return authClient.rpcClient.Call(serviceMethod, args, reply)
|
||||
@@ -139,6 +171,7 @@ func (authClient *AuthRPCClient) call(serviceMethod string, args interface {
|
||||
// Call executes RPC call till success or globalAuthRPCRetryThreshold on ErrShutdown.
|
||||
func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
|
||||
SetAuthToken(authToken string)
|
||||
SetRPCAPIVersion(version semVersion)
|
||||
}, reply interface{}) (err error) {
|
||||
|
||||
// Done channel is used to close any lingering retry routine, as soon
|
||||
@@ -159,6 +192,11 @@ func (authClient *AuthRPCClient) Call(serviceMethod string, args interface {
|
||||
}
|
||||
}
|
||||
}
|
||||
// gob doesn't provide any typed errors for us to reflect
|
||||
// upon, this is the only way to return proper error.
|
||||
if err != nil && strings.Contains(err.Error(), "gob: wrong type") {
|
||||
err = errRPCAPIVersionUnsupported
|
||||
}
|
||||
break
|
||||
}
|
||||
return err
|
||||
@@ -169,6 +207,10 @@ func (authClient *AuthRPCClient) Close() error {
|
||||
authClient.Lock()
|
||||
defer authClient.Unlock()
|
||||
|
||||
if authClient.rpcClient == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
authClient.authToken = ""
|
||||
return authClient.rpcClient.Close()
|
||||
}
|
||||
@@ -182,3 +224,87 @@ func (authClient *AuthRPCClient) ServerAddr() string {
|
||||
func (authClient *AuthRPCClient) ServiceEndpoint() string {
|
||||
return authClient.config.serviceEndpoint
|
||||
}
|
||||
|
||||
// default Dial timeout for RPC connections.
|
||||
const defaultDialTimeout = 3 * time.Second
|
||||
|
||||
// Connect success message required from rpc server.
|
||||
const connectSuccessMessage = "200 Connected to Go RPC"
|
||||
|
||||
// dial tries to establish a connection to serverAddr in a safe manner.
|
||||
// If there is a valid rpc.Cliemt, it returns that else creates a new one.
|
||||
func rpcDial(serverAddr, serviceEndpoint string, secureConn bool) (netRPCClient *rpc.Client, err error) {
|
||||
if serverAddr == "" || serviceEndpoint == "" {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
d := &net.Dialer{
|
||||
Timeout: defaultDialTimeout,
|
||||
}
|
||||
var conn net.Conn
|
||||
if secureConn {
|
||||
var hostname string
|
||||
if hostname, _, err = net.SplitHostPort(serverAddr); err != nil {
|
||||
return nil, &net.OpError{
|
||||
Op: "dial-http",
|
||||
Net: serverAddr + serviceEndpoint,
|
||||
Addr: nil,
|
||||
Err: fmt.Errorf("Unable to parse server address <%s>: %s", serverAddr, err),
|
||||
}
|
||||
}
|
||||
// ServerName in tls.Config needs to be specified to support SNI certificates.
|
||||
conn, err = tls.DialWithDialer(d, "tcp", serverAddr, &tls.Config{
|
||||
ServerName: hostname,
|
||||
RootCAs: globalRootCAs,
|
||||
})
|
||||
} else {
|
||||
conn, err = d.Dial("tcp", serverAddr)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
// Print RPC connection errors that are worthy to display in log.
|
||||
switch err.(type) {
|
||||
case x509.HostnameError:
|
||||
errorIf(err, "Unable to establish secure connection to %s", serverAddr)
|
||||
}
|
||||
|
||||
return nil, &net.OpError{
|
||||
Op: "dial-http",
|
||||
Net: serverAddr + serviceEndpoint,
|
||||
Addr: nil,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
// Check for network errors writing over the dialed conn.
|
||||
if _, err = io.WriteString(conn, "CONNECT "+serviceEndpoint+" HTTP/1.0\n\n"); err != nil {
|
||||
conn.Close()
|
||||
return nil, &net.OpError{
|
||||
Op: "dial-http",
|
||||
Net: serverAddr + serviceEndpoint,
|
||||
Addr: nil,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to read the HTTP response for the HTTP method CONNECT, upon
|
||||
// success return the RPC connection instance.
|
||||
resp, err := http.ReadResponse(bufio.NewReader(conn), &http.Request{
|
||||
Method: http.MethodConnect,
|
||||
})
|
||||
if err != nil {
|
||||
conn.Close()
|
||||
return nil, &net.OpError{
|
||||
Op: "dial-http",
|
||||
Net: serverAddr + serviceEndpoint,
|
||||
Addr: nil,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
if resp.Status != connectSuccessMessage {
|
||||
conn.Close()
|
||||
return nil, errors.New("unexpected HTTP response: " + resp.Status)
|
||||
}
|
||||
|
||||
// Initialize rpc client.
|
||||
return rpc.NewClient(conn), nil
|
||||
}
|
||||
|
||||
@@ -16,7 +16,12 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"crypto/x509"
|
||||
"os"
|
||||
"path"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Tests authorized RPC client.
|
||||
func TestAuthRPCClient(t *testing.T) {
|
||||
@@ -53,3 +58,81 @@ func TestAuthRPCClient(t *testing.T) {
|
||||
t.Fatalf("Unexpected node value %s, but expected %s", authRPC.ServiceEndpoint(), authCfg.serviceEndpoint)
|
||||
}
|
||||
}
|
||||
|
||||
// Test rpc dial test.
|
||||
func TestRPCDial(t *testing.T) {
|
||||
prevRootCAs := globalRootCAs
|
||||
defer func() {
|
||||
globalRootCAs = prevRootCAs
|
||||
}()
|
||||
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
testServer := StartTestServer(t, "")
|
||||
defer testServer.Stop()
|
||||
|
||||
cert, key, err := generateTLSCertKey("127.0.0.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set global root CAs.
|
||||
globalRootCAs = x509.NewCertPool()
|
||||
globalRootCAs.AppendCertsFromPEM(cert)
|
||||
|
||||
testServerTLS := StartTestTLSServer(t, "", cert, key)
|
||||
defer testServerTLS.Stop()
|
||||
|
||||
adminEndpoint := path.Join(minioReservedBucketPath, adminPath)
|
||||
testCases := []struct {
|
||||
serverAddr string
|
||||
serverEndpoint string
|
||||
success bool
|
||||
secure bool
|
||||
}{
|
||||
// Empty server addr should fail.
|
||||
{
|
||||
serverAddr: "",
|
||||
serverEndpoint: adminEndpoint,
|
||||
success: false,
|
||||
},
|
||||
// Unexpected server addr should fail.
|
||||
{
|
||||
serverAddr: "example.com",
|
||||
serverEndpoint: adminEndpoint,
|
||||
success: false,
|
||||
},
|
||||
// Server addr connects but fails for CONNECT call.
|
||||
{
|
||||
serverAddr: "example.com:80",
|
||||
serverEndpoint: "/",
|
||||
success: false,
|
||||
},
|
||||
// Successful connecting to insecure RPC server.
|
||||
{
|
||||
serverAddr: testServer.Server.Listener.Addr().String(),
|
||||
serverEndpoint: adminEndpoint,
|
||||
success: true,
|
||||
},
|
||||
// Successful connecting to secure RPC server.
|
||||
{
|
||||
serverAddr: testServerTLS.Server.Listener.Addr().String(),
|
||||
serverEndpoint: adminEndpoint,
|
||||
success: true,
|
||||
secure: true,
|
||||
},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
_, err = rpcDial(testCase.serverAddr, testCase.serverEndpoint, testCase.secure)
|
||||
if err != nil && testCase.success {
|
||||
t.Errorf("Test %d: Expected success but found failure instead %s", i+1, err)
|
||||
}
|
||||
if err == nil && !testCase.success {
|
||||
t.Errorf("Test %d: Expected failure but found success instead", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,7 @@ func TestLogin(t *testing.T) {
|
||||
t.Fatalf("Failed to create test config - %v", err)
|
||||
}
|
||||
defer os.RemoveAll(rootPath)
|
||||
creds := serverConfig.GetCredential()
|
||||
creds := globalServerConfig.GetCredential()
|
||||
token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
@@ -43,7 +43,7 @@ func TestLogin(t *testing.T) {
|
||||
{
|
||||
args: LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
},
|
||||
skewTime: 0,
|
||||
expectedErr: nil,
|
||||
@@ -52,16 +52,16 @@ func TestLogin(t *testing.T) {
|
||||
{
|
||||
args: LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: "INVALID-" + Version,
|
||||
Version: semVersion{2, 0, 0},
|
||||
},
|
||||
skewTime: 0,
|
||||
expectedErr: errServerVersionMismatch,
|
||||
expectedErr: errRPCAPIVersionUnsupported,
|
||||
},
|
||||
// Valid username, password and version, not request time
|
||||
{
|
||||
args: LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
},
|
||||
skewTime: 20 * time.Minute,
|
||||
expectedErr: errServerTimeMismatch,
|
||||
@@ -70,7 +70,7 @@ func TestLogin(t *testing.T) {
|
||||
{
|
||||
args: LoginRPCArgs{
|
||||
AuthToken: "",
|
||||
Version: Version,
|
||||
Version: globalRPCAPIVersion,
|
||||
},
|
||||
skewTime: 0,
|
||||
expectedErr: errAuthentication,
|
||||
|
||||
@@ -239,7 +239,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
var buffer = new(bytes.Buffer)
|
||||
err = obj.GetObject(bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer)
|
||||
err = obj.GetObject(bucket, "object"+strconv.Itoa(i%10), 0, int64(objSize), buffer, "")
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
@@ -402,7 +402,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
b.RunParallel(func(pb *testing.PB) {
|
||||
i := 0
|
||||
for pb.Next() {
|
||||
err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard)
|
||||
err = obj.GetObject(bucket, "object"+strconv.Itoa(i), 0, int64(objSize), ioutil.Discard, "")
|
||||
if err != nil {
|
||||
b.Error(err)
|
||||
}
|
||||
|
||||
@@ -50,13 +50,17 @@ func (br *browserPeerAPIHandlers) SetAuthPeer(args SetAuthPeerArgs, reply *AuthR
|
||||
return fmt.Errorf("Invalid credential passed")
|
||||
}
|
||||
|
||||
// Acquire lock before updating global configuration.
|
||||
globalServerConfigMu.Lock()
|
||||
defer globalServerConfigMu.Unlock()
|
||||
|
||||
// Update credentials in memory
|
||||
prevCred := serverConfig.SetCredential(args.Creds)
|
||||
prevCred := globalServerConfig.SetCredential(args.Creds)
|
||||
|
||||
// Save credentials to config file
|
||||
if err := serverConfig.Save(); err != nil {
|
||||
if err := globalServerConfig.Save(); err != nil {
|
||||
// Save the current creds when failed to update.
|
||||
serverConfig.SetCredential(prevCred)
|
||||
globalServerConfig.SetCredential(prevCred)
|
||||
|
||||
errorIf(err, "Unable to update the config with new credentials sent from browser RPC.")
|
||||
return err
|
||||
@@ -77,7 +81,7 @@ func updateCredsOnPeers(creds auth.Credentials) map[string]error {
|
||||
errs := make([]error, len(peers))
|
||||
var wg sync.WaitGroup
|
||||
|
||||
serverCred := serverConfig.GetCredential()
|
||||
serverCred := globalServerConfig.GetCredential()
|
||||
// Launch go routines to send request to each peer in parallel.
|
||||
for ix := range peers {
|
||||
wg.Add(1)
|
||||
|
||||
@@ -70,11 +70,13 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
|
||||
|
||||
// Validate for invalid token.
|
||||
args := SetAuthPeerArgs{Creds: creds}
|
||||
args.AuthToken = "garbage"
|
||||
rclient := newRPCClient(s.testAuthConf.serverAddr, s.testAuthConf.serviceEndpoint, false)
|
||||
rclient := newAuthRPCClient(s.testAuthConf)
|
||||
defer rclient.Close()
|
||||
err = rclient.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{})
|
||||
if err != nil {
|
||||
if err = rclient.Login(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rclient.authToken = "garbage"
|
||||
if err = rclient.Call("BrowserPeer.SetAuthPeer", &args, &AuthRPCReply{}); err != nil {
|
||||
if err.Error() != errInvalidToken.Error() {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -90,20 +92,14 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
|
||||
}
|
||||
|
||||
// Validate for failure in login handler with previous credentials.
|
||||
rclient = newRPCClient(s.testAuthConf.serverAddr, s.testAuthConf.serviceEndpoint, false)
|
||||
rclient = newAuthRPCClient(s.testAuthConf)
|
||||
defer rclient.Close()
|
||||
token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rargs := &LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
rreply := &LoginRPCReply{}
|
||||
err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply)
|
||||
if err != nil {
|
||||
rclient.authToken = token
|
||||
if err = rclient.Login(); err != nil {
|
||||
if err.Error() != errInvalidAccessKeyID.Error() {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -113,14 +109,8 @@ func (s *TestRPCBrowserPeerSuite) testBrowserPeerRPC(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Validate for success in loing handled with valid credetnails.
|
||||
rargs = &LoginRPCArgs{
|
||||
AuthToken: token,
|
||||
Version: Version,
|
||||
RequestTime: UTCNow(),
|
||||
}
|
||||
rreply = &LoginRPCReply{}
|
||||
if err = rclient.Call("BrowserPeer"+loginMethodName, rargs, rreply); err != nil {
|
||||
rclient.authToken = token
|
||||
if err = rclient.Login(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
router "github.com/gorilla/mux"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// Set up an RPC endpoint that receives browser related calls. The
|
||||
@@ -40,7 +42,7 @@ func registerBrowserPeerRPCRouter(mux *router.Router) error {
|
||||
bpRPCServer := newRPCServer()
|
||||
err := bpRPCServer.RegisterName("BrowserPeer", bpHandlers)
|
||||
if err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
bpRouter := mux.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter()
|
||||
|
||||
@@ -44,7 +44,7 @@ func validateListObjectsArgs(prefix, marker, delimiter string, maxKeys int) APIE
|
||||
if marker != "" {
|
||||
// Marker not common with prefix is not implemented.
|
||||
if !hasPrefix(marker, prefix) {
|
||||
return ErrNotImplemented
|
||||
return ErrInvalidPrefixMarker
|
||||
}
|
||||
}
|
||||
// Success.
|
||||
@@ -89,7 +89,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -107,15 +107,23 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
|
||||
// Validate the query params before beginning to serve the request.
|
||||
// fetch-owner is not validated since it is a boolean
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
|
||||
// return empty response if invalid marker
|
||||
//TODO: avoid this pattern when moving to errors package
|
||||
if s3Error == ErrInvalidPrefixMarker {
|
||||
listObjectsInfo := ListObjectsInfo{}
|
||||
response := generateListObjectsV2Response(bucket, prefix, token, marker, startAfter, delimiter, fetchOwner, listObjectsInfo.IsTruncated, maxKeys, listObjectsInfo.Objects, listObjectsInfo.Prefixes)
|
||||
writeSuccessResponseXML(w, encodeResponse(response))
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Inititate a list objects operation based on the input params.
|
||||
// On success would return back ListObjectsInfo object to be
|
||||
// marshalled into S3 compatible XML header.
|
||||
listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list objects.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -142,7 +150,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -152,6 +160,13 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
|
||||
// Validate all the query params before beginning to serve the request.
|
||||
if s3Error := validateListObjectsArgs(prefix, marker, delimiter, maxKeys); s3Error != ErrNone {
|
||||
// return empty response if invalid marker
|
||||
//TODO: avoid this pattern when moving to errors package
|
||||
if s3Error == ErrInvalidPrefixMarker {
|
||||
response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, maxKeys, ListObjectsInfo{})
|
||||
writeSuccessResponseXML(w, encodeResponse(response))
|
||||
return
|
||||
}
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -161,7 +176,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
|
||||
// marshalled into S3 compatible XML header.
|
||||
listObjectsInfo, err := objectAPI.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list objects.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
+14
-44
@@ -32,6 +32,7 @@ import (
|
||||
mux "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/set"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
@@ -40,7 +41,7 @@ import (
|
||||
func enforceBucketPolicy(bucket, action, resource, referer, sourceIP string, queryParams url.Values) (s3Error APIErrorCode) {
|
||||
// Verify if bucket actually exists
|
||||
if err := checkBucketExist(bucket, newObjectLayerFn()); err != nil {
|
||||
err = errorCause(err)
|
||||
err = errors.Cause(err)
|
||||
switch err.(type) {
|
||||
case BucketNameInvalid:
|
||||
// Return error for invalid bucket name.
|
||||
@@ -119,7 +120,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
s3Error := checkRequestAuthType(r, bucket, "s3:GetBucketLocation", globalMinioDefaultRegion)
|
||||
if s3Error == ErrInvalidRegion {
|
||||
// Clients like boto3 send getBucketLocation() call signed with region that is configured.
|
||||
s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", serverConfig.GetRegion())
|
||||
s3Error = checkRequestAuthType(r, "", "s3:GetBucketLocation", globalServerConfig.GetRegion())
|
||||
}
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
@@ -127,7 +128,6 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
|
||||
errorIf(err, "Unable to fetch bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -135,7 +135,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
|
||||
// Generate response.
|
||||
encodedSuccessResponse := encodeResponse(LocationResponse{})
|
||||
// Get current region.
|
||||
region := serverConfig.GetRegion()
|
||||
region := globalServerConfig.GetRegion()
|
||||
if region != globalMinioDefaultRegion {
|
||||
encodedSuccessResponse = encodeResponse(LocationResponse{
|
||||
Location: region,
|
||||
@@ -164,7 +164,7 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucketMultipartUploads", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -184,7 +184,6 @@ func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter,
|
||||
|
||||
listMultipartsInfo, err := objectAPI.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list multipart uploads.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -211,7 +210,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
s3Error := checkRequestAuthType(r, "", "", globalMinioDefaultRegion)
|
||||
if s3Error == ErrInvalidRegion {
|
||||
// Clients like boto3 send listBuckets() call signed with region that is configured.
|
||||
s3Error = checkRequestAuthType(r, "", "", serverConfig.GetRegion())
|
||||
s3Error = checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
|
||||
}
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
@@ -220,7 +219,6 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
// Invoke the list buckets.
|
||||
bucketsInfo, err := objectAPI.ListBuckets()
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list buckets.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -245,7 +243,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
var authError APIErrorCode
|
||||
if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", serverConfig.GetRegion()); authError != ErrNone {
|
||||
if authError = checkRequestAuthType(r, bucket, "s3:DeleteObject", globalServerConfig.GetRegion()); authError != ErrNone {
|
||||
// In the event access is denied, a 200 response should still be returned
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html
|
||||
if authError != ErrAccessDenied {
|
||||
@@ -303,16 +301,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}
|
||||
return
|
||||
}
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, obj.ObjectName)
|
||||
if timedOutErr := objectLock.GetLock(globalObjectTimeout); timedOutErr != nil {
|
||||
dErrs[i] = timedOutErr
|
||||
} else {
|
||||
defer objectLock.Unlock()
|
||||
|
||||
dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
|
||||
if dErr != nil {
|
||||
dErrs[i] = dErr
|
||||
}
|
||||
dErr := objectAPI.DeleteObject(bucket, obj.ObjectName)
|
||||
if dErr != nil {
|
||||
dErrs[i] = dErr
|
||||
}
|
||||
}(index, object)
|
||||
}
|
||||
@@ -328,13 +319,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
continue
|
||||
}
|
||||
if _, ok := errorCause(err).(ObjectNotFound); ok {
|
||||
if _, ok := errors.Cause(err).(ObjectNotFound); ok {
|
||||
// If the object is not found it should be
|
||||
// accounted as deleted as per S3 spec.
|
||||
deletedObjects = append(deletedObjects, object)
|
||||
continue
|
||||
}
|
||||
errorIf(err, "Unable to delete object. %s", object.ObjectName)
|
||||
// Error during delete should be collected separately.
|
||||
deleteErrors = append(deleteErrors, DeleteError{
|
||||
Code: errorCodeResponse[toAPIErrorCode(err)].Code,
|
||||
@@ -384,7 +374,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
|
||||
// PutBucket does not have any bucket action.
|
||||
s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion())
|
||||
s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
@@ -407,17 +397,9 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
return
|
||||
}
|
||||
|
||||
bucketLock := globalNSMutex.NewNSLock(bucket, "")
|
||||
if bucketLock.GetLock(globalObjectTimeout) != nil {
|
||||
writeErrorResponse(w, ErrOperationTimedOut, r.URL)
|
||||
return
|
||||
}
|
||||
defer bucketLock.Unlock()
|
||||
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucketWithLocation(bucket, "")
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to create a bucket.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -545,13 +527,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
lengthRange := postPolicyForm.Conditions.ContentLengthRange
|
||||
if lengthRange.Valid {
|
||||
if fileSize < lengthRange.Min {
|
||||
errorIf(err, "Unable to create object.")
|
||||
writeErrorResponse(w, toAPIErrorCode(errDataTooSmall), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
|
||||
errorIf(err, "Unable to create object.")
|
||||
writeErrorResponse(w, toAPIErrorCode(errDataTooLarge), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -565,13 +545,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, object)
|
||||
if objectLock.GetLock(globalObjectTimeout) != nil {
|
||||
writeErrorResponse(w, ErrOperationTimedOut, r.URL)
|
||||
return
|
||||
}
|
||||
defer objectLock.Unlock()
|
||||
|
||||
hashReader, err := hash.NewReader(fileBody, fileSize, "", "")
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to initialize hashReader.")
|
||||
@@ -581,7 +554,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
objInfo, err := objectAPI.PutObject(bucket, object, hashReader, metadata)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to create object.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -648,7 +620,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, bucket, "s3:ListBucket", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponseHeadersOnly(w, s3Error)
|
||||
return
|
||||
}
|
||||
@@ -661,7 +633,6 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
defer bucketLock.RUnlock()
|
||||
|
||||
if _, err := objectAPI.GetBucketInfo(bucket); err != nil {
|
||||
errorIf(err, "Unable to fetch bucket info.")
|
||||
writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
|
||||
return
|
||||
}
|
||||
@@ -678,7 +649,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
// DeleteBucket does not have any bucket action.
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -695,7 +666,6 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
|
||||
// Attempt to delete bucket.
|
||||
if err := objectAPI.DeleteBucket(bucket); err != nil {
|
||||
errorIf(err, "Unable to delete a bucket.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -209,5 +209,5 @@ type arnSQS struct {
|
||||
|
||||
// Stringer for constructing AWS ARN compatible string.
|
||||
func (m arnSQS) String() string {
|
||||
return minioSqs + serverConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
|
||||
return minioSqs + globalServerConfig.GetRegion() + ":" + m.AccountID + ":" + m.Type
|
||||
}
|
||||
|
||||
@@ -28,6 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -48,7 +49,7 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -58,20 +59,19 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
|
||||
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Attempt to successfully load notification config.
|
||||
nConfig, err := loadNotificationConfig(bucket, objAPI)
|
||||
if err != nil && errorCause(err) != errNoSuchNotifications {
|
||||
if err != nil && errors.Cause(err) != errNoSuchNotifications {
|
||||
errorIf(err, "Unable to read notification configuration.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
// For no notifications we write a dummy XML.
|
||||
if errorCause(err) == errNoSuchNotifications {
|
||||
if errors.Cause(err) == errNoSuchNotifications {
|
||||
// Complies with the s3 behavior in this regard.
|
||||
nConfig = ¬ificationConfig{}
|
||||
}
|
||||
@@ -102,7 +102,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -112,7 +112,6 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
|
||||
|
||||
_, err := objectAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -296,7 +295,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -327,7 +326,6 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to get bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -337,7 +335,7 @@ func (api objectAPIHandlers) ListenBucketNotificationHandler(w http.ResponseWrit
|
||||
accountARN := fmt.Sprintf(
|
||||
"%s:%s:%s:%s-%s",
|
||||
minioTopic,
|
||||
serverConfig.GetRegion(),
|
||||
globalServerConfig.GetRegion(),
|
||||
accountID,
|
||||
snsTypeMinio,
|
||||
targetServer,
|
||||
|
||||
@@ -120,7 +120,7 @@ func checkARN(arn, arnType string) APIErrorCode {
|
||||
// Server region is allowed to be empty by default,
|
||||
// in such a scenario ARN region is not validating
|
||||
// allowing all regions.
|
||||
if sregion := serverConfig.GetRegion(); sregion != "" {
|
||||
if sregion := globalServerConfig.GetRegion(); sregion != "" {
|
||||
region := strs[3]
|
||||
if region != sregion {
|
||||
return ErrRegionNotification
|
||||
@@ -146,34 +146,34 @@ func isValidQueueID(queueARN string) bool {
|
||||
// Is Queue identifier valid?.
|
||||
|
||||
if isAMQPQueue(sqsARN) { // AMQP eueue.
|
||||
amqpN := serverConfig.Notify.GetAMQPByID(sqsARN.AccountID)
|
||||
amqpN := globalServerConfig.Notify.GetAMQPByID(sqsARN.AccountID)
|
||||
return amqpN.Enable && amqpN.URL != ""
|
||||
} else if isMQTTQueue(sqsARN) {
|
||||
mqttN := serverConfig.Notify.GetMQTTByID(sqsARN.AccountID)
|
||||
mqttN := globalServerConfig.Notify.GetMQTTByID(sqsARN.AccountID)
|
||||
return mqttN.Enable && mqttN.Broker != ""
|
||||
} else if isNATSQueue(sqsARN) {
|
||||
natsN := serverConfig.Notify.GetNATSByID(sqsARN.AccountID)
|
||||
natsN := globalServerConfig.Notify.GetNATSByID(sqsARN.AccountID)
|
||||
return natsN.Enable && natsN.Address != ""
|
||||
} else if isElasticQueue(sqsARN) { // Elastic queue.
|
||||
elasticN := serverConfig.Notify.GetElasticSearchByID(sqsARN.AccountID)
|
||||
elasticN := globalServerConfig.Notify.GetElasticSearchByID(sqsARN.AccountID)
|
||||
return elasticN.Enable && elasticN.URL != ""
|
||||
} else if isRedisQueue(sqsARN) { // Redis queue.
|
||||
redisN := serverConfig.Notify.GetRedisByID(sqsARN.AccountID)
|
||||
redisN := globalServerConfig.Notify.GetRedisByID(sqsARN.AccountID)
|
||||
return redisN.Enable && redisN.Addr != ""
|
||||
} else if isPostgreSQLQueue(sqsARN) {
|
||||
pgN := serverConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
|
||||
pgN := globalServerConfig.Notify.GetPostgreSQLByID(sqsARN.AccountID)
|
||||
// Postgres can work with only default conn. info.
|
||||
return pgN.Enable
|
||||
} else if isMySQLQueue(sqsARN) {
|
||||
msqlN := serverConfig.Notify.GetMySQLByID(sqsARN.AccountID)
|
||||
msqlN := globalServerConfig.Notify.GetMySQLByID(sqsARN.AccountID)
|
||||
// Mysql can work with only default conn. info.
|
||||
return msqlN.Enable
|
||||
} else if isKafkaQueue(sqsARN) {
|
||||
kafkaN := serverConfig.Notify.GetKafkaByID(sqsARN.AccountID)
|
||||
kafkaN := globalServerConfig.Notify.GetKafkaByID(sqsARN.AccountID)
|
||||
return (kafkaN.Enable && len(kafkaN.Brokers) > 0 &&
|
||||
kafkaN.Topic != "")
|
||||
} else if isWebhookQueue(sqsARN) {
|
||||
webhookN := serverConfig.Notify.GetWebhookByID(sqsARN.AccountID)
|
||||
webhookN := globalServerConfig.Notify.GetWebhookByID(sqsARN.AccountID)
|
||||
return webhookN.Enable && webhookN.Endpoint != ""
|
||||
}
|
||||
return false
|
||||
@@ -281,7 +281,7 @@ func unmarshalSqsARN(queueARN string) (mSqs arnSQS) {
|
||||
// Server region is allowed to be empty by default,
|
||||
// in such a scenario ARN region is not validating
|
||||
// allowing all regions.
|
||||
if sregion := serverConfig.GetRegion(); sregion != "" {
|
||||
if sregion := globalServerConfig.GetRegion(); sregion != "" {
|
||||
region := strs[3]
|
||||
if region != sregion {
|
||||
return
|
||||
|
||||
@@ -225,7 +225,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -236,7 +236,6 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -284,7 +283,7 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -295,7 +294,6 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -325,7 +323,7 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -336,7 +334,6 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -344,7 +341,6 @@ func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
// Read bucket access policy.
|
||||
policy, err := readBucketPolicy(bucket, objAPI)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
+8
-30
@@ -24,6 +24,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
@@ -87,8 +88,7 @@ func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.Bucke
|
||||
// List buckets to proceed loading all notification configuration.
|
||||
buckets, err := objAPI.ListBuckets()
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list buckets.")
|
||||
return nil, errorCause(err)
|
||||
return nil, errors.Cause(err)
|
||||
}
|
||||
|
||||
policies = make(map[string]policy.BucketAccessPolicy)
|
||||
@@ -99,7 +99,7 @@ func loadAllBucketPolicies(objAPI ObjectLayer) (policies map[string]policy.Bucke
|
||||
if pErr != nil {
|
||||
// net.Dial fails for rpc client or any
|
||||
// other unexpected errors during net.Dial.
|
||||
if !isErrIgnored(pErr, errDiskNotFound) {
|
||||
if !errors.IsErrIgnored(pErr, errDiskNotFound) {
|
||||
if !isErrBucketPolicyNotFound(pErr) {
|
||||
pErrs = append(pErrs, pErr)
|
||||
}
|
||||
@@ -148,21 +148,14 @@ func initBucketPolicies(objAPI ObjectLayer) error {
|
||||
func readBucketPolicyJSON(bucket string, objAPI ObjectLayer) (bucketPolicyReader io.Reader, err error) {
|
||||
policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
|
||||
|
||||
// Acquire a read lock on policy config before reading.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
|
||||
if err = objLock.GetRLock(globalOperationTimeout); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer objLock.RUnlock()
|
||||
|
||||
var buffer bytes.Buffer
|
||||
err = objAPI.GetObject(minioMetaBucket, policyPath, 0, -1, &buffer)
|
||||
err = objAPI.GetObject(minioMetaBucket, policyPath, 0, -1, &buffer, "")
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
|
||||
return nil, BucketPolicyNotFound{Bucket: bucket}
|
||||
}
|
||||
errorIf(err, "Unable to load policy for the bucket %s.", bucket)
|
||||
return nil, errorCause(err)
|
||||
return nil, errors.Cause(err)
|
||||
}
|
||||
|
||||
return &buffer, nil
|
||||
@@ -190,16 +183,9 @@ func readBucketPolicy(bucket string, objAPI ObjectLayer) (policy.BucketAccessPol
|
||||
// if no policies are found.
|
||||
func removeBucketPolicy(bucket string, objAPI ObjectLayer) error {
|
||||
policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
|
||||
// Acquire a write lock on policy config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
err := objAPI.DeleteObject(minioMetaBucket, policyPath)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to remove bucket-policy on bucket %s.", bucket)
|
||||
err = errorCause(err)
|
||||
err = errors.Cause(err)
|
||||
if _, ok := err.(ObjectNotFound); ok {
|
||||
return BucketPolicyNotFound{Bucket: bucket}
|
||||
}
|
||||
@@ -216,22 +202,15 @@ func writeBucketPolicy(bucket string, objAPI ObjectLayer, bpy policy.BucketAcces
|
||||
return err
|
||||
}
|
||||
policyPath := pathJoin(bucketConfigPrefix, bucket, bucketPolicyConfig)
|
||||
// Acquire a write lock on policy config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, policyPath)
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to set policy for the bucket %s", bucket)
|
||||
return errorCause(err)
|
||||
return errors.Cause(err)
|
||||
}
|
||||
|
||||
if _, err = objAPI.PutObject(minioMetaBucket, policyPath, hashReader, nil); err != nil {
|
||||
errorIf(err, "Unable to set policy for the bucket %s", bucket)
|
||||
return errorCause(err)
|
||||
return errors.Cause(err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -241,7 +220,6 @@ func parseAndPersistBucketPolicy(bucket string, policyBytes []byte, objAPI Objec
|
||||
var bktPolicy policy.BucketAccessPolicy
|
||||
err := parseBucketPolicy(bytes.NewReader(policyBytes), &bktPolicy)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to parse bucket policy.")
|
||||
return ErrInvalidPolicyDocument
|
||||
}
|
||||
|
||||
|
||||
@@ -16,11 +16,13 @@
|
||||
|
||||
package cmd
|
||||
|
||||
import "go/build"
|
||||
|
||||
// DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants
|
||||
// set through ‘buildscripts/gen-ldflags.go’.
|
||||
var (
|
||||
// GOPATH - GOPATH value at the time of build.
|
||||
GOPATH = ""
|
||||
GOPATH = build.Default.GOPATH
|
||||
|
||||
// Go get development tag.
|
||||
goGetTag = "DEVELOPMENT.GOGET"
|
||||
|
||||
+37
-1
@@ -22,9 +22,15 @@ import (
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
// TLSPrivateKeyPassword is the environment variable which contains the password used
|
||||
// to decrypt the TLS private key. It must be set if the TLS private key is
|
||||
// password protected.
|
||||
const TLSPrivateKeyPassword = "MINIO_CERT_PASSWD"
|
||||
|
||||
func parsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) {
|
||||
// Read certificate file.
|
||||
var data []byte
|
||||
@@ -90,6 +96,36 @@ func getRootCAs(certsCAsDir string) (*x509.CertPool, error) {
|
||||
return rootCAs, nil
|
||||
}
|
||||
|
||||
// load an X509 key pair (private key , certificate) from the provided
|
||||
// paths. The private key may be encrypted and is decrypted using the
|
||||
// ENV_VAR: MINIO_CERT_PASSWD.
|
||||
func loadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) {
|
||||
certPEMBlock, err := ioutil.ReadFile(certFile)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("TLS: failed to read cert file: %v", err)
|
||||
}
|
||||
keyPEMBlock, err := ioutil.ReadFile(keyFile)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("TLS: failed to read private key: %v", err)
|
||||
}
|
||||
key, rest := pem.Decode(keyPEMBlock)
|
||||
if len(rest) > 0 {
|
||||
return tls.Certificate{}, fmt.Errorf("TLS: private key contains additional data")
|
||||
}
|
||||
if x509.IsEncryptedPEMBlock(key) {
|
||||
password, ok := os.LookupEnv(TLSPrivateKeyPassword)
|
||||
if !ok {
|
||||
return tls.Certificate{}, fmt.Errorf("TLS: private key is encrypted but no password is present - set env var: %s", TLSPrivateKeyPassword)
|
||||
}
|
||||
decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password))
|
||||
if decErr != nil {
|
||||
return tls.Certificate{}, fmt.Errorf("TLS: failed to decrypt private key: %v", decErr)
|
||||
}
|
||||
keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey})
|
||||
}
|
||||
return tls.X509KeyPair(certPEMBlock, keyPEMBlock)
|
||||
}
|
||||
|
||||
func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsCert *tls.Certificate, secureConn bool, err error) {
|
||||
if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
|
||||
return nil, nil, nil, false, nil
|
||||
@@ -100,7 +136,7 @@ func getSSLConfig() (x509Certs []*x509.Certificate, rootCAs *x509.CertPool, tlsC
|
||||
}
|
||||
|
||||
var cert tls.Certificate
|
||||
if cert, err = tls.LoadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
|
||||
if cert, err = loadX509KeyPair(getPublicCertFile(), getPrivateKeyFile()); err != nil {
|
||||
return nil, nil, nil, false, err
|
||||
}
|
||||
|
||||
|
||||
@@ -257,3 +257,252 @@ func TestGetRootCAs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadX509KeyPair(t *testing.T) {
|
||||
for i, testCase := range loadX509KeyPairTests {
|
||||
privateKey, err := createTempFile("private.key", testCase.privateKey)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to create tmp private key file: %v", i, err)
|
||||
}
|
||||
certificate, err := createTempFile("public.crt", testCase.certificate)
|
||||
if err != nil {
|
||||
os.Remove(privateKey)
|
||||
t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err)
|
||||
}
|
||||
|
||||
os.Unsetenv(TLSPrivateKeyPassword)
|
||||
if testCase.password != "" {
|
||||
os.Setenv(TLSPrivateKeyPassword, testCase.password)
|
||||
}
|
||||
_, err = loadX509KeyPair(certificate, privateKey)
|
||||
if err != nil && !testCase.shouldFail {
|
||||
t.Errorf("Test %d: test should succeed but it failed: %v", i, err)
|
||||
}
|
||||
if err == nil && testCase.shouldFail {
|
||||
t.Errorf("Test %d: test should fail but it succeed", i)
|
||||
}
|
||||
os.Remove(privateKey)
|
||||
os.Remove(certificate)
|
||||
}
|
||||
}
|
||||
|
||||
var loadX509KeyPairTests = []struct {
|
||||
password string
|
||||
privateKey, certificate string
|
||||
shouldFail bool
|
||||
}{
|
||||
{
|
||||
password: "foobar",
|
||||
privateKey: `-----BEGIN RSA PRIVATE KEY-----
|
||||
Proc-Type: 4,ENCRYPTED
|
||||
DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
|
||||
|
||||
nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
|
||||
AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
|
||||
uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
|
||||
cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
|
||||
q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
|
||||
lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
|
||||
bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
|
||||
IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
|
||||
i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
|
||||
r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
|
||||
ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
|
||||
dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
|
||||
ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
|
||||
piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
|
||||
6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
|
||||
GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
|
||||
HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
|
||||
KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
|
||||
UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
|
||||
ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
|
||||
37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
|
||||
zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
|
||||
btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
|
||||
9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
|
||||
aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
|
||||
-----END RSA PRIVATE KEY-----`,
|
||||
certificate: `-----BEGIN CERTIFICATE-----
|
||||
MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
|
||||
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
|
||||
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
|
||||
MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
|
||||
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
|
||||
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
|
||||
AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
|
||||
HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
|
||||
GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
|
||||
VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
|
||||
Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
|
||||
cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
|
||||
0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
|
||||
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
|
||||
iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
|
||||
b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
|
||||
KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
|
||||
bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
|
||||
kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
|
||||
-----END CERTIFICATE-----`,
|
||||
shouldFail: false,
|
||||
},
|
||||
{
|
||||
password: "password",
|
||||
privateKey: `-----BEGIN RSA PRIVATE KEY-----
|
||||
Proc-Type: 4,ENCRYPTED
|
||||
DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285
|
||||
|
||||
nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb
|
||||
AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h
|
||||
uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j
|
||||
cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj
|
||||
q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog
|
||||
lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB
|
||||
bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU
|
||||
IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO
|
||||
i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1
|
||||
r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ
|
||||
ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh
|
||||
dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA
|
||||
ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8
|
||||
piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3
|
||||
6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL
|
||||
GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy
|
||||
HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz
|
||||
KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg
|
||||
UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL
|
||||
ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo
|
||||
37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi
|
||||
zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx
|
||||
btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx
|
||||
9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B
|
||||
aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k
|
||||
-----END RSA PRIVATE KEY-----`,
|
||||
certificate: `-----BEGIN CERTIFICATE-----
|
||||
MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
|
||||
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
|
||||
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
|
||||
MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
|
||||
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
|
||||
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
|
||||
AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6
|
||||
HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR
|
||||
GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc
|
||||
VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla
|
||||
Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd
|
||||
cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD
|
||||
0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV
|
||||
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy
|
||||
iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ
|
||||
b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj
|
||||
KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb
|
||||
bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl
|
||||
kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR
|
||||
-----END CERTIFICATE-----`,
|
||||
shouldFail: true,
|
||||
},
|
||||
{
|
||||
password: "",
|
||||
privateKey: `-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
|
||||
WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
|
||||
Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
|
||||
6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
|
||||
QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
|
||||
mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
|
||||
+PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
|
||||
qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
|
||||
1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
|
||||
VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
|
||||
Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
|
||||
hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
|
||||
FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
|
||||
92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
|
||||
hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
|
||||
yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
|
||||
eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
|
||||
M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
|
||||
ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
|
||||
8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
|
||||
qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
|
||||
wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
|
||||
JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
|
||||
+jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
|
||||
0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
|
||||
-----END RSA PRIVATE KEY-----`,
|
||||
certificate: `-----BEGIN CERTIFICATE-----
|
||||
MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
|
||||
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
|
||||
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
|
||||
NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
|
||||
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
|
||||
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
|
||||
AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
|
||||
ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
|
||||
EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
|
||||
z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
|
||||
zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
|
||||
r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
|
||||
d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
|
||||
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
|
||||
Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
|
||||
SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
|
||||
5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
|
||||
1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
|
||||
OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
|
||||
-----END CERTIFICATE-----`,
|
||||
shouldFail: false,
|
||||
},
|
||||
{
|
||||
password: "foobar",
|
||||
privateKey: `-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw
|
||||
WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6
|
||||
Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM
|
||||
6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg
|
||||
QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP
|
||||
mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ
|
||||
+PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd
|
||||
qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4
|
||||
1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3
|
||||
VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa
|
||||
Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI
|
||||
hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T
|
||||
FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz
|
||||
92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO
|
||||
hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF
|
||||
yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna
|
||||
eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM
|
||||
M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i
|
||||
ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX
|
||||
8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1
|
||||
qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K
|
||||
wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5
|
||||
JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk
|
||||
+jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW
|
||||
0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg==
|
||||
-----END RSA PRIVATE KEY-----`,
|
||||
certificate: `-----BEGIN CERTIFICATE-----
|
||||
MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV
|
||||
BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV
|
||||
BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4
|
||||
NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0
|
||||
YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw
|
||||
EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB
|
||||
AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9
|
||||
ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz
|
||||
EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr
|
||||
z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq
|
||||
zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI
|
||||
r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0
|
||||
d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV
|
||||
HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+
|
||||
Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw
|
||||
SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd
|
||||
5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt
|
||||
1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO
|
||||
OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD
|
||||
-----END CERTIFICATE-----`,
|
||||
shouldFail: false,
|
||||
},
|
||||
}
|
||||
|
||||
+47
-24
@@ -29,35 +29,21 @@ import (
|
||||
|
||||
// Check for updates and print a notification message
|
||||
func checkUpdate(mode string) {
|
||||
// Its OK to ignore any errors during getUpdateInfo() here.
|
||||
if older, downloadURL, err := getUpdateInfo(1*time.Second, mode); err == nil {
|
||||
if updateMsg := computeUpdateMessage(downloadURL, older); updateMsg != "" {
|
||||
// Its OK to ignore any errors during doUpdate() here.
|
||||
if updateMsg, _, currentReleaseTime, latestReleaseTime, err := getUpdateInfo(2*time.Second, mode); err == nil {
|
||||
if globalInplaceUpdateDisabled {
|
||||
log.Println(updateMsg)
|
||||
} else {
|
||||
log.Println(prepareUpdateMessage("Run `minio update`", latestReleaseTime.Sub(currentReleaseTime)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func enableLoggers() {
|
||||
fileLogTarget := serverConfig.Logger.GetFile()
|
||||
if fileLogTarget.Enable {
|
||||
err := InitFileLogger(&fileLogTarget)
|
||||
fatalIf(err, "Unable to initialize file logger")
|
||||
log.AddTarget(fileLogTarget)
|
||||
}
|
||||
|
||||
consoleLogTarget := serverConfig.Logger.GetConsole()
|
||||
if consoleLogTarget.Enable {
|
||||
InitConsoleLogger(&consoleLogTarget)
|
||||
}
|
||||
|
||||
log.SetConsoleTarget(consoleLogTarget)
|
||||
}
|
||||
|
||||
func initConfig() {
|
||||
// Config file does not exist, we create it fresh and return upon success.
|
||||
if isFile(getConfigFile()) {
|
||||
fatalIf(migrateConfig(), "Config migration failed.")
|
||||
fatalIf(loadConfig(), "Unable to load config version: '%s'.", v20)
|
||||
fatalIf(loadConfig(), "Unable to load config version: '%s'.", serverConfigVersion)
|
||||
} else {
|
||||
fatalIf(newConfig(), "Unable to initialize minio config for the first time.")
|
||||
log.Println("Created minio configuration file successfully at " + getConfigDir())
|
||||
@@ -90,9 +76,6 @@ func handleCommonEnvVars() {
|
||||
globalProfiler = startProfiler(profiler)
|
||||
}
|
||||
|
||||
// Check if object cache is disabled.
|
||||
globalXLObjCacheDisabled = strings.EqualFold(os.Getenv("_MINIO_CACHE"), "off")
|
||||
|
||||
accessKey := os.Getenv("MINIO_ACCESS_KEY")
|
||||
secretKey := os.Getenv("MINIO_SECRET_KEY")
|
||||
if accessKey != "" && secretKey != "" {
|
||||
@@ -116,10 +99,50 @@ func handleCommonEnvVars() {
|
||||
globalIsBrowserEnabled = bool(browserFlag)
|
||||
}
|
||||
|
||||
globalHTTPTrace = os.Getenv("MINIO_HTTP_TRACE") != ""
|
||||
traceFile := os.Getenv("MINIO_HTTP_TRACE")
|
||||
if traceFile != "" {
|
||||
var err error
|
||||
globalHTTPTraceFile, err = os.OpenFile(traceFile, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0660)
|
||||
fatalIf(err, "error opening file %s", traceFile)
|
||||
}
|
||||
|
||||
globalDomainName = os.Getenv("MINIO_DOMAIN")
|
||||
if globalDomainName != "" {
|
||||
globalIsEnvDomainName = true
|
||||
}
|
||||
|
||||
// In place update is true by default if the MINIO_UPDATE is not set
|
||||
// or is not set to 'off', if MINIO_UPDATE is set to 'off' then
|
||||
// in-place update is off.
|
||||
globalInplaceUpdateDisabled = strings.EqualFold(os.Getenv("MINIO_UPDATE"), "off")
|
||||
|
||||
// Validate and store the storage class env variables only for XL/Dist XL setups
|
||||
if globalIsXL {
|
||||
var err error
|
||||
|
||||
// Check for environment variables and parse into storageClass struct
|
||||
if ssc := os.Getenv(standardStorageClassEnv); ssc != "" {
|
||||
globalStandardStorageClass, err = parseStorageClass(ssc)
|
||||
fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
|
||||
}
|
||||
|
||||
if rrsc := os.Getenv(reducedRedundancyStorageClassEnv); rrsc != "" {
|
||||
globalRRStorageClass, err = parseStorageClass(rrsc)
|
||||
fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
|
||||
}
|
||||
|
||||
// Validation is done after parsing both the storage classes. This is needed because we need one
|
||||
// storage class value to deduce the correct value of the other storage class.
|
||||
if globalRRStorageClass.Scheme != "" {
|
||||
err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
|
||||
fatalIf(err, "Invalid value set in environment variable %s.", reducedRedundancyStorageClassEnv)
|
||||
globalIsStorageClass = true
|
||||
}
|
||||
|
||||
if globalStandardStorageClass.Scheme != "" {
|
||||
err = validateParity(globalStandardStorageClass.Parity, globalRRStorageClass.Parity)
|
||||
fatalIf(err, "Invalid value set in environment variable %s.", standardStorageClassEnv)
|
||||
globalIsStorageClass = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
@@ -27,64 +28,43 @@ import (
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
// Steps to move from version N to version N+1
|
||||
// 1. Add new struct serverConfigVN+1 in config-versions.go
|
||||
// 2. Set serverConfigVersion to "N+1"
|
||||
// 3. Set serverConfig to serverConfigVN+1
|
||||
// 4. Add new migration function (ex. func migrateVNToVN+1()) in config-migrate.go
|
||||
// 5. Call migrateVNToVN+1() from migrateConfig() in config-migrate.go
|
||||
// 6. Make changes in config-current_test.go for any test change
|
||||
|
||||
// Config version
|
||||
const v20 = "20"
|
||||
const serverConfigVersion = "22"
|
||||
|
||||
type serverConfig = serverConfigV22
|
||||
|
||||
var (
|
||||
// serverConfig server config.
|
||||
serverConfig *serverConfigV20
|
||||
serverConfigMu sync.RWMutex
|
||||
// globalServerConfig server config.
|
||||
globalServerConfig *serverConfig
|
||||
globalServerConfigMu sync.RWMutex
|
||||
)
|
||||
|
||||
// serverConfigV20 server configuration version '20' which is like
|
||||
// version '19' except it adds support for VirtualHostDomain
|
||||
type serverConfigV20 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// GetVersion get current config version.
|
||||
func (s *serverConfigV20) GetVersion() string {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
func (s *serverConfig) GetVersion() string {
|
||||
return s.Version
|
||||
}
|
||||
|
||||
// SetRegion set a new region.
|
||||
func (s *serverConfigV20) SetRegion(region string) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
func (s *serverConfig) SetRegion(region string) {
|
||||
// Save new region.
|
||||
s.Region = region
|
||||
}
|
||||
|
||||
// GetRegion get current region.
|
||||
func (s *serverConfigV20) GetRegion() string {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
func (s *serverConfig) GetRegion() string {
|
||||
return s.Region
|
||||
}
|
||||
|
||||
// SetCredentials set new credentials. SetCredential returns the previous credential.
|
||||
func (s *serverConfigV20) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
// SetCredential sets new credential and returns the previous credential.
|
||||
func (s *serverConfig) SetCredential(creds auth.Credentials) (prevCred auth.Credentials) {
|
||||
// Save previous credential.
|
||||
prevCred = s.Credential
|
||||
|
||||
@@ -96,52 +76,95 @@ func (s *serverConfigV20) SetCredential(creds auth.Credentials) (prevCred auth.C
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s *serverConfigV20) GetCredential() auth.Credentials {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
func (s *serverConfig) GetCredential() auth.Credentials {
|
||||
return s.Credential
|
||||
}
|
||||
|
||||
// SetBrowser set if browser is enabled.
|
||||
func (s *serverConfigV20) SetBrowser(b bool) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
func (s *serverConfig) SetBrowser(b bool) {
|
||||
// Set the new value.
|
||||
s.Browser = BrowserFlag(b)
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s *serverConfigV20) GetBrowser() bool {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
func (s *serverConfig) SetStorageClass(standardClass, rrsClass storageClass) {
|
||||
s.StorageClass.Standard = standardClass
|
||||
s.StorageClass.RRS = rrsClass
|
||||
}
|
||||
|
||||
// GetStorageClass reads storage class fields from current config.
|
||||
// It returns the standard and reduced redundancy storage class struct
|
||||
func (s *serverConfig) GetStorageClass() (storageClass, storageClass) {
|
||||
return s.StorageClass.Standard, s.StorageClass.RRS
|
||||
}
|
||||
|
||||
// GetCredentials get current credentials.
|
||||
func (s *serverConfig) GetBrowser() bool {
|
||||
return bool(s.Browser)
|
||||
}
|
||||
|
||||
// Save config.
|
||||
func (s *serverConfigV20) Save() error {
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
|
||||
func (s *serverConfig) Save() error {
|
||||
// Save config file.
|
||||
return quick.Save(getConfigFile(), s)
|
||||
}
|
||||
|
||||
func newServerConfigV20() *serverConfigV20 {
|
||||
srvCfg := &serverConfigV20{
|
||||
Version: v20,
|
||||
// Returns the string describing a difference with the given
|
||||
// configuration object. If the given configuration object is
|
||||
// identical, an empty string is returned.
|
||||
func (s *serverConfig) ConfigDiff(t *serverConfig) string {
|
||||
switch {
|
||||
case t == nil:
|
||||
return "Given configuration is empty"
|
||||
case s.Credential != t.Credential:
|
||||
return "Credential configuration differs"
|
||||
case s.Region != t.Region:
|
||||
return "Region configuration differs"
|
||||
case s.Browser != t.Browser:
|
||||
return "Browser configuration differs"
|
||||
case s.Domain != t.Domain:
|
||||
return "Domain configuration differs"
|
||||
case s.StorageClass != t.StorageClass:
|
||||
return "StorageClass configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.AMQP, t.Notify.AMQP):
|
||||
return "AMQP Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.NATS, t.Notify.NATS):
|
||||
return "NATS Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.ElasticSearch, t.Notify.ElasticSearch):
|
||||
return "ElasticSearch Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.Redis, t.Notify.Redis):
|
||||
return "Redis Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.PostgreSQL, t.Notify.PostgreSQL):
|
||||
return "PostgreSQL Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.Kafka, t.Notify.Kafka):
|
||||
return "Kafka Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.Webhook, t.Notify.Webhook):
|
||||
return "Webhook Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.MySQL, t.Notify.MySQL):
|
||||
return "MySQL Notification configuration differs"
|
||||
case !reflect.DeepEqual(s.Notify.MQTT, t.Notify.MQTT):
|
||||
return "MQTT Notification configuration differs"
|
||||
case reflect.DeepEqual(s, t):
|
||||
return ""
|
||||
default:
|
||||
// This case will not happen unless this comparison
|
||||
// function has become stale.
|
||||
return "Configuration differs"
|
||||
}
|
||||
}
|
||||
|
||||
func newServerConfig() *serverConfig {
|
||||
srvCfg := &serverConfig{
|
||||
Version: serverConfigVersion,
|
||||
Credential: auth.MustGetNewCredentials(),
|
||||
Region: globalMinioDefaultRegion,
|
||||
Browser: true,
|
||||
Logger: &loggers{},
|
||||
Notify: ¬ifier{},
|
||||
StorageClass: storageClassConfig{
|
||||
Standard: storageClass{},
|
||||
RRS: storageClass{},
|
||||
},
|
||||
Notify: notifier{},
|
||||
}
|
||||
|
||||
// Enable console logger by default on a fresh run.
|
||||
srvCfg.Logger.Console = NewConsoleLogger()
|
||||
|
||||
// Make sure to initialize notification configs.
|
||||
srvCfg.Notify.AMQP = make(map[string]amqpNotify)
|
||||
srvCfg.Notify.AMQP["1"] = amqpNotify{}
|
||||
@@ -169,7 +192,7 @@ func newServerConfigV20() *serverConfigV20 {
|
||||
// found, otherwise use default parameters
|
||||
func newConfig() error {
|
||||
// Initialize server config.
|
||||
srvCfg := newServerConfigV20()
|
||||
srvCfg := newServerConfig()
|
||||
|
||||
// If env is set override the credentials from config file.
|
||||
if globalIsEnvCreds {
|
||||
@@ -188,15 +211,19 @@ func newConfig() error {
|
||||
srvCfg.Domain = globalDomainName
|
||||
}
|
||||
|
||||
if globalIsStorageClass {
|
||||
srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
// Save the new config globally.
|
||||
// unlock the mutex.
|
||||
serverConfigMu.Lock()
|
||||
serverConfig = srvCfg
|
||||
serverConfigMu.Unlock()
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
globalServerConfigMu.Unlock()
|
||||
|
||||
// Save config into file.
|
||||
return serverConfig.Save()
|
||||
return globalServerConfig.Save()
|
||||
}
|
||||
|
||||
// doCheckDupJSONKeys recursively detects duplicate json keys
|
||||
@@ -251,8 +278,8 @@ func checkDupJSONKeys(json string) error {
|
||||
}
|
||||
|
||||
// getValidConfig - returns valid server configuration
|
||||
func getValidConfig() (*serverConfigV20, error) {
|
||||
srvCfg := &serverConfigV20{
|
||||
func getValidConfig() (*serverConfig, error) {
|
||||
srvCfg := &serverConfig{
|
||||
Region: globalMinioDefaultRegion,
|
||||
Browser: true,
|
||||
}
|
||||
@@ -262,8 +289,8 @@ func getValidConfig() (*serverConfigV20, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if srvCfg.Version != v20 {
|
||||
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", v20, srvCfg.Version)
|
||||
if srvCfg.Version != serverConfigVersion {
|
||||
return nil, fmt.Errorf("configuration version mismatch. Expected: ‘%s’, Got: ‘%s’", serverConfigVersion, srvCfg.Version)
|
||||
}
|
||||
|
||||
// Load config file json and check for duplication json keys
|
||||
@@ -283,11 +310,6 @@ func getValidConfig() (*serverConfigV20, error) {
|
||||
return nil, errors.New("invalid credential in config file " + configFile)
|
||||
}
|
||||
|
||||
// Validate logger field
|
||||
if err = srvCfg.Logger.Validate(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Validate notify field
|
||||
if err = srvCfg.Notify.Validate(); err != nil {
|
||||
return nil, err
|
||||
@@ -321,22 +343,29 @@ func loadConfig() error {
|
||||
srvCfg.Domain = globalDomainName
|
||||
}
|
||||
|
||||
if globalIsStorageClass {
|
||||
srvCfg.SetStorageClass(globalStandardStorageClass, globalRRStorageClass)
|
||||
}
|
||||
|
||||
// hold the mutex lock before a new config is assigned.
|
||||
serverConfigMu.Lock()
|
||||
serverConfig = srvCfg
|
||||
globalServerConfigMu.Lock()
|
||||
globalServerConfig = srvCfg
|
||||
if !globalIsEnvCreds {
|
||||
globalActiveCred = serverConfig.GetCredential()
|
||||
globalActiveCred = globalServerConfig.GetCredential()
|
||||
}
|
||||
if !globalIsEnvBrowser {
|
||||
globalIsBrowserEnabled = serverConfig.GetBrowser()
|
||||
globalIsBrowserEnabled = globalServerConfig.GetBrowser()
|
||||
}
|
||||
if !globalIsEnvRegion {
|
||||
globalServerRegion = serverConfig.GetRegion()
|
||||
globalServerRegion = globalServerConfig.GetRegion()
|
||||
}
|
||||
if !globalIsEnvDomainName {
|
||||
globalDomainName = serverConfig.Domain
|
||||
globalDomainName = globalServerConfig.Domain
|
||||
}
|
||||
serverConfigMu.Unlock()
|
||||
if !globalIsStorageClass {
|
||||
globalStandardStorageClass, globalRRStorageClass = globalServerConfig.GetStorageClass()
|
||||
}
|
||||
globalServerConfigMu.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/tidwall/gjson"
|
||||
)
|
||||
|
||||
@@ -34,95 +35,72 @@ func TestServerConfig(t *testing.T) {
|
||||
// remove the root directory after the test ends.
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
if serverConfig.GetRegion() != globalMinioDefaultRegion {
|
||||
t.Errorf("Expecting region `us-east-1` found %s", serverConfig.GetRegion())
|
||||
if globalServerConfig.GetRegion() != globalMinioDefaultRegion {
|
||||
t.Errorf("Expecting region `us-east-1` found %s", globalServerConfig.GetRegion())
|
||||
}
|
||||
|
||||
// Set new region and verify.
|
||||
serverConfig.SetRegion("us-west-1")
|
||||
if serverConfig.GetRegion() != "us-west-1" {
|
||||
t.Errorf("Expecting region `us-west-1` found %s", serverConfig.GetRegion())
|
||||
globalServerConfig.SetRegion("us-west-1")
|
||||
if globalServerConfig.GetRegion() != "us-west-1" {
|
||||
t.Errorf("Expecting region `us-west-1` found %s", globalServerConfig.GetRegion())
|
||||
}
|
||||
|
||||
// Set new amqp notification id.
|
||||
serverConfig.Notify.SetAMQPByID("2", amqpNotify{})
|
||||
savedNotifyCfg1 := serverConfig.Notify.GetAMQPByID("2")
|
||||
globalServerConfig.Notify.SetAMQPByID("2", amqpNotify{})
|
||||
savedNotifyCfg1 := globalServerConfig.Notify.GetAMQPByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg1, amqpNotify{}) {
|
||||
t.Errorf("Expecting AMQP config %#v found %#v", amqpNotify{}, savedNotifyCfg1)
|
||||
}
|
||||
|
||||
// Set new elastic search notification id.
|
||||
serverConfig.Notify.SetElasticSearchByID("2", elasticSearchNotify{})
|
||||
savedNotifyCfg2 := serverConfig.Notify.GetElasticSearchByID("2")
|
||||
globalServerConfig.Notify.SetElasticSearchByID("2", elasticSearchNotify{})
|
||||
savedNotifyCfg2 := globalServerConfig.Notify.GetElasticSearchByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg2, elasticSearchNotify{}) {
|
||||
t.Errorf("Expecting Elasticsearch config %#v found %#v", elasticSearchNotify{}, savedNotifyCfg2)
|
||||
}
|
||||
|
||||
// Set new redis notification id.
|
||||
serverConfig.Notify.SetRedisByID("2", redisNotify{})
|
||||
savedNotifyCfg3 := serverConfig.Notify.GetRedisByID("2")
|
||||
globalServerConfig.Notify.SetRedisByID("2", redisNotify{})
|
||||
savedNotifyCfg3 := globalServerConfig.Notify.GetRedisByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg3, redisNotify{}) {
|
||||
t.Errorf("Expecting Redis config %#v found %#v", redisNotify{}, savedNotifyCfg3)
|
||||
}
|
||||
|
||||
// Set new kafka notification id.
|
||||
serverConfig.Notify.SetKafkaByID("2", kafkaNotify{})
|
||||
savedNotifyCfg4 := serverConfig.Notify.GetKafkaByID("2")
|
||||
globalServerConfig.Notify.SetKafkaByID("2", kafkaNotify{})
|
||||
savedNotifyCfg4 := globalServerConfig.Notify.GetKafkaByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg4, kafkaNotify{}) {
|
||||
t.Errorf("Expecting Kafka config %#v found %#v", kafkaNotify{}, savedNotifyCfg4)
|
||||
}
|
||||
|
||||
// Set new Webhook notification id.
|
||||
serverConfig.Notify.SetWebhookByID("2", webhookNotify{})
|
||||
savedNotifyCfg5 := serverConfig.Notify.GetWebhookByID("2")
|
||||
globalServerConfig.Notify.SetWebhookByID("2", webhookNotify{})
|
||||
savedNotifyCfg5 := globalServerConfig.Notify.GetWebhookByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg5, webhookNotify{}) {
|
||||
t.Errorf("Expecting Webhook config %#v found %#v", webhookNotify{}, savedNotifyCfg5)
|
||||
}
|
||||
|
||||
// Set new console logger.
|
||||
// Set new MySQL notification id.
|
||||
serverConfig.Notify.SetMySQLByID("2", mySQLNotify{})
|
||||
savedNotifyCfg6 := serverConfig.Notify.GetMySQLByID("2")
|
||||
globalServerConfig.Notify.SetMySQLByID("2", mySQLNotify{})
|
||||
savedNotifyCfg6 := globalServerConfig.Notify.GetMySQLByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg6, mySQLNotify{}) {
|
||||
t.Errorf("Expecting Webhook config %#v found %#v", mySQLNotify{}, savedNotifyCfg6)
|
||||
}
|
||||
|
||||
// Set new console logger.
|
||||
// Set new MQTT notification id.
|
||||
serverConfig.Notify.SetMQTTByID("2", mqttNotify{})
|
||||
savedNotifyCfg7 := serverConfig.Notify.GetMQTTByID("2")
|
||||
globalServerConfig.Notify.SetMQTTByID("2", mqttNotify{})
|
||||
savedNotifyCfg7 := globalServerConfig.Notify.GetMQTTByID("2")
|
||||
if !reflect.DeepEqual(savedNotifyCfg7, mqttNotify{}) {
|
||||
t.Errorf("Expecting Webhook config %#v found %#v", mqttNotify{}, savedNotifyCfg7)
|
||||
}
|
||||
|
||||
consoleLogger := NewConsoleLogger()
|
||||
serverConfig.Logger.SetConsole(consoleLogger)
|
||||
consoleCfg := serverConfig.Logger.GetConsole()
|
||||
if !reflect.DeepEqual(consoleCfg, consoleLogger) {
|
||||
t.Errorf("Expecting console logger config %#v found %#v", consoleLogger, consoleCfg)
|
||||
}
|
||||
// Set new console logger.
|
||||
consoleLogger.Enable = false
|
||||
serverConfig.Logger.SetConsole(consoleLogger)
|
||||
|
||||
// Set new file logger.
|
||||
fileLogger := NewFileLogger("test-log-file")
|
||||
serverConfig.Logger.SetFile(fileLogger)
|
||||
fileCfg := serverConfig.Logger.GetFile()
|
||||
if !reflect.DeepEqual(fileCfg, fileLogger) {
|
||||
t.Errorf("Expecting file logger config %#v found %#v", fileLogger, fileCfg)
|
||||
}
|
||||
// Set new file logger.
|
||||
fileLogger.Enable = false
|
||||
serverConfig.Logger.SetFile(fileLogger)
|
||||
|
||||
// Match version.
|
||||
if serverConfig.GetVersion() != v20 {
|
||||
t.Errorf("Expecting version %s found %s", serverConfig.GetVersion(), v20)
|
||||
if globalServerConfig.GetVersion() != serverConfigVersion {
|
||||
t.Errorf("Expecting version %s found %s", globalServerConfig.GetVersion(), serverConfigVersion)
|
||||
}
|
||||
|
||||
// Attempt to save.
|
||||
if err := serverConfig.Save(); err != nil {
|
||||
if err := globalServerConfig.Save(); err != nil {
|
||||
t.Fatalf("Unable to save updated config file %s", err)
|
||||
}
|
||||
|
||||
@@ -172,17 +150,17 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
defer os.RemoveAll(rootPath)
|
||||
|
||||
// Check if serverConfig has
|
||||
if serverConfig.GetBrowser() {
|
||||
t.Errorf("Expecting browser is set to false found %v", serverConfig.GetBrowser())
|
||||
if globalServerConfig.GetBrowser() {
|
||||
t.Errorf("Expecting browser is set to false found %v", globalServerConfig.GetBrowser())
|
||||
}
|
||||
|
||||
// Check if serverConfig has
|
||||
if serverConfig.GetRegion() != "us-west-1" {
|
||||
t.Errorf("Expecting region to be \"us-west-1\" found %v", serverConfig.GetRegion())
|
||||
if globalServerConfig.GetRegion() != "us-west-1" {
|
||||
t.Errorf("Expecting region to be \"us-west-1\" found %v", globalServerConfig.GetRegion())
|
||||
}
|
||||
|
||||
// Check if serverConfig has
|
||||
cred := serverConfig.GetCredential()
|
||||
cred := globalServerConfig.GetCredential()
|
||||
|
||||
if cred.AccessKey != "minio" {
|
||||
t.Errorf("Expecting access key to be `minio` found %s", cred.AccessKey)
|
||||
@@ -192,8 +170,8 @@ func TestServerConfigWithEnvs(t *testing.T) {
|
||||
t.Errorf("Expecting access key to be `minio123` found %s", cred.SecretKey)
|
||||
}
|
||||
|
||||
if serverConfig.Domain != "domain.com" {
|
||||
t.Errorf("Expecting Domain to be `domain.com` found " + serverConfig.Domain)
|
||||
if globalServerConfig.Domain != "domain.com" {
|
||||
t.Errorf("Expecting Domain to be `domain.com` found " + globalServerConfig.Domain)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,7 +215,7 @@ func TestValidateConfig(t *testing.T) {
|
||||
|
||||
configPath := filepath.Join(rootPath, minioConfigFile)
|
||||
|
||||
v := v20
|
||||
v := serverConfigVersion
|
||||
|
||||
testCases := []struct {
|
||||
configData string
|
||||
@@ -273,58 +251,55 @@ func TestValidateConfig(t *testing.T) {
|
||||
// Test 10 - duplicated json keys
|
||||
{`{"version": "` + v + `", "browser": "on", "browser": "on", "region":"us-east-1", "credential" : {"accessKey":"minio", "secretKey":"minio123"}}`, false},
|
||||
|
||||
// Test 11 - empty filename field in File
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "logger": { "file": { "enable": true, "filename": "" } }}`, false},
|
||||
|
||||
// Test 12 - Test AMQP
|
||||
// Test 11 - Test AMQP
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "amqp": { "1": { "enable": true, "url": "", "exchange": "", "routingKey": "", "exchangeType": "", "mandatory": false, "immediate": false, "durable": false, "internal": false, "noWait": false, "autoDeleted": false }}}}`, false},
|
||||
|
||||
// Test 13 - Test NATS
|
||||
// Test 12 - Test NATS
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nats": { "1": { "enable": true, "address": "", "subject": "", "username": "", "password": "", "token": "", "secure": false, "pingInterval": 0, "streaming": { "enable": false, "clusterID": "", "clientID": "", "async": false, "maxPubAcksInflight": 0 } } }}}`, false},
|
||||
|
||||
// Test 14 - Test ElasticSearch
|
||||
// Test 13 - Test ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "url": "", "index": "" } }}}`, false},
|
||||
|
||||
// Test 15 - Test Redis
|
||||
// Test 14 - Test Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "address": "", "password": "", "key": "" } }}}`, false},
|
||||
|
||||
// Test 16 - Test PostgreSQL
|
||||
// Test 15 - Test PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
// Test 17 - Test Kafka
|
||||
// Test 16 - Test Kafka
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "kafka": { "1": { "enable": true, "brokers": null, "topic": "" } }}}`, false},
|
||||
|
||||
// Test 18 - Test Webhook
|
||||
// Test 17 - Test Webhook
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "webhook": { "1": { "enable": true, "endpoint": "" } }}}`, false},
|
||||
|
||||
// Test 20 - Test MySQL
|
||||
// Test 18 - Test MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "table": "", "host": "", "port": "", "user": "", "password": "", "database": "" }}}}`, false},
|
||||
|
||||
// Test 21 - Test Format for MySQL
|
||||
// Test 19 - Test Format for MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "format": "invalid", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
|
||||
|
||||
// Test 22 - Test valid Format for MySQL
|
||||
// Test 20 - Test valid Format for MySQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mysql": { "1": { "enable": true, "dsnString": "", "format": "namespace", "table": "xxx", "host": "10.0.0.1", "port": "3306", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
|
||||
// Test 23 - Test Format for PostgreSQL
|
||||
// Test 21 - Test Format for PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "invalid", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, false},
|
||||
|
||||
// Test 24 - Test valid Format for PostgreSQL
|
||||
// Test 22 - Test valid Format for PostgreSQL
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "postgresql": { "1": { "enable": true, "connectionString": "", "format": "namespace", "table": "xxx", "host": "myhost", "port": "5432", "user": "abc", "password": "pqr", "database": "test1" }}}}`, true},
|
||||
|
||||
// Test 25 - Test Format for ElasticSearch
|
||||
// Test 23 - Test Format for ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "invalid", "url": "example.com", "index": "myindex" } }}}`, false},
|
||||
|
||||
// Test 26 - Test valid Format for ElasticSearch
|
||||
// Test 24 - Test valid Format for ElasticSearch
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "elasticsearch": { "1": { "enable": true, "format": "namespace", "url": "example.com", "index": "myindex" } }}}`, true},
|
||||
|
||||
// Test 27 - Test Format for Redis
|
||||
// Test 25 - Test Format for Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "invalid", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, false},
|
||||
|
||||
// Test 28 - Test valid Format for Redis
|
||||
// Test 26 - Test valid Format for Redis
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
|
||||
|
||||
// Test 29 - Test MQTT
|
||||
// Test 27 - Test MQTT
|
||||
{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "", "topic": "", "qos": 0, "clientId": "", "username": "", "password": ""}}}}`, false},
|
||||
}
|
||||
|
||||
@@ -337,8 +312,97 @@ func TestValidateConfig(t *testing.T) {
|
||||
t.Errorf("Test %d, should pass but it failed with err = %v", i+1, verr)
|
||||
}
|
||||
if !testCase.shouldPass && verr == nil {
|
||||
t.Errorf("Test %d, should fail but it succeed.", i+1)
|
||||
t.Errorf("Test %d, should fail but it succeeded.", i+1)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestConfigDiff(t *testing.T) {
|
||||
testCases := []struct {
|
||||
s, t *serverConfig
|
||||
diff string
|
||||
}{
|
||||
// 1
|
||||
{&serverConfig{}, nil, "Given configuration is empty"},
|
||||
// 2
|
||||
{
|
||||
&serverConfig{Credential: auth.Credentials{"u1", "p1"}},
|
||||
&serverConfig{Credential: auth.Credentials{"u1", "p2"}},
|
||||
"Credential configuration differs",
|
||||
},
|
||||
// 3
|
||||
{&serverConfig{Region: "us-east-1"}, &serverConfig{Region: "us-west-1"}, "Region configuration differs"},
|
||||
// 4
|
||||
{&serverConfig{Browser: false}, &serverConfig{Browser: true}, "Browser configuration differs"},
|
||||
// 5
|
||||
{&serverConfig{Domain: "domain1"}, &serverConfig{Domain: "domain2"}, "Domain configuration differs"},
|
||||
// 6
|
||||
{
|
||||
&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 6}}},
|
||||
&serverConfig{StorageClass: storageClassConfig{storageClass{"1", 8}, storageClass{"2", 4}}},
|
||||
"StorageClass configuration differs",
|
||||
},
|
||||
// 7
|
||||
{
|
||||
&serverConfig{Notify: notifier{AMQP: map[string]amqpNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{AMQP: map[string]amqpNotify{"1": {Enable: false}}}},
|
||||
"AMQP Notification configuration differs",
|
||||
},
|
||||
// 8
|
||||
{
|
||||
&serverConfig{Notify: notifier{NATS: map[string]natsNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{NATS: map[string]natsNotify{"1": {Enable: false}}}},
|
||||
"NATS Notification configuration differs",
|
||||
},
|
||||
// 9
|
||||
{
|
||||
&serverConfig{Notify: notifier{ElasticSearch: map[string]elasticSearchNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{ElasticSearch: map[string]elasticSearchNotify{"1": {Enable: false}}}},
|
||||
"ElasticSearch Notification configuration differs",
|
||||
},
|
||||
// 10
|
||||
{
|
||||
&serverConfig{Notify: notifier{Redis: map[string]redisNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{Redis: map[string]redisNotify{"1": {Enable: false}}}},
|
||||
"Redis Notification configuration differs",
|
||||
},
|
||||
// 11
|
||||
{
|
||||
&serverConfig{Notify: notifier{PostgreSQL: map[string]postgreSQLNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{PostgreSQL: map[string]postgreSQLNotify{"1": {Enable: false}}}},
|
||||
"PostgreSQL Notification configuration differs",
|
||||
},
|
||||
// 12
|
||||
{
|
||||
&serverConfig{Notify: notifier{Kafka: map[string]kafkaNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{Kafka: map[string]kafkaNotify{"1": {Enable: false}}}},
|
||||
"Kafka Notification configuration differs",
|
||||
},
|
||||
// 13
|
||||
{
|
||||
&serverConfig{Notify: notifier{Webhook: map[string]webhookNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{Webhook: map[string]webhookNotify{"1": {Enable: false}}}},
|
||||
"Webhook Notification configuration differs",
|
||||
},
|
||||
// 14
|
||||
{
|
||||
&serverConfig{Notify: notifier{MySQL: map[string]mySQLNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{MySQL: map[string]mySQLNotify{"1": {Enable: false}}}},
|
||||
"MySQL Notification configuration differs",
|
||||
},
|
||||
// 15
|
||||
{
|
||||
&serverConfig{Notify: notifier{MQTT: map[string]mqttNotify{"1": {Enable: true}}}},
|
||||
&serverConfig{Notify: notifier{MQTT: map[string]mqttNotify{"1": {Enable: false}}}},
|
||||
"MQTT Notification configuration differs",
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
got := testCase.s.ConfigDiff(testCase.t)
|
||||
if got != testCase.diff {
|
||||
t.Errorf("Test %d: got %s expected %s", i+1, got, testCase.diff)
|
||||
}
|
||||
}
|
||||
}
|
||||
+224
-9
@@ -143,7 +143,7 @@ func migrateConfig() error {
|
||||
}
|
||||
fallthrough
|
||||
case "18":
|
||||
// Migrate version '17' to '18'.
|
||||
// Migrate version '18' to '19'.
|
||||
if err = migrateV18ToV19(); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -154,6 +154,15 @@ func migrateConfig() error {
|
||||
}
|
||||
fallthrough
|
||||
case "20":
|
||||
if err = migrateV20ToV21(); err != nil {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case "21":
|
||||
if err = migrateV21ToV22(); err != nil {
|
||||
return err
|
||||
}
|
||||
case serverConfigVersion:
|
||||
// No migration needed. this always points to current version.
|
||||
err = nil
|
||||
}
|
||||
@@ -1138,14 +1147,6 @@ func migrateV15ToV16() error {
|
||||
// Load browser config from existing config in the file.
|
||||
srvConfig.Browser = cv15.Browser
|
||||
|
||||
// Migrate console and file fields
|
||||
if cv15.Logger.Console.Enable {
|
||||
srvConfig.Logger.Console = NewConsoleLogger()
|
||||
}
|
||||
if cv15.Logger.File.Enable {
|
||||
srvConfig.Logger.File = NewFileLogger(cv15.Logger.File.Filename)
|
||||
}
|
||||
|
||||
if err = quick.Save(configFile, srvConfig); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv15.Version, srvConfig.Version, err)
|
||||
}
|
||||
@@ -1592,3 +1593,217 @@ func migrateV19ToV20() error {
|
||||
log.Printf(configMigrateMSGTemplate, configFile, cv19.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV20ToV21() error {
|
||||
configFile := getConfigFile()
|
||||
|
||||
cv20 := &serverConfigV20{}
|
||||
_, err := quick.Load(configFile, cv20)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config version ‘20’. %v", err)
|
||||
}
|
||||
if cv20.Version != "20" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy over fields from V20 into V21 config struct
|
||||
srvConfig := &serverConfigV21{
|
||||
Notify: ¬ifier{},
|
||||
}
|
||||
srvConfig.Version = "21"
|
||||
srvConfig.Credential = cv20.Credential
|
||||
srvConfig.Region = cv20.Region
|
||||
if srvConfig.Region == "" {
|
||||
// Region needs to be set for AWS Signature Version 4.
|
||||
srvConfig.Region = globalMinioDefaultRegion
|
||||
}
|
||||
|
||||
// check and set notifiers config
|
||||
if len(cv20.Notify.AMQP) == 0 {
|
||||
srvConfig.Notify.AMQP = make(map[string]amqpNotify)
|
||||
srvConfig.Notify.AMQP["1"] = amqpNotify{}
|
||||
} else {
|
||||
// New deliveryMode parameter is added for AMQP,
|
||||
// default value is already 0, so nothing to
|
||||
// explicitly migrate here.
|
||||
srvConfig.Notify.AMQP = cv20.Notify.AMQP
|
||||
}
|
||||
if len(cv20.Notify.ElasticSearch) == 0 {
|
||||
srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
|
||||
srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.ElasticSearch = cv20.Notify.ElasticSearch
|
||||
}
|
||||
if len(cv20.Notify.Redis) == 0 {
|
||||
srvConfig.Notify.Redis = make(map[string]redisNotify)
|
||||
srvConfig.Notify.Redis["1"] = redisNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.Redis = cv20.Notify.Redis
|
||||
}
|
||||
if len(cv20.Notify.PostgreSQL) == 0 {
|
||||
srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
|
||||
srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.PostgreSQL = cv20.Notify.PostgreSQL
|
||||
}
|
||||
if len(cv20.Notify.Kafka) == 0 {
|
||||
srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
|
||||
srvConfig.Notify.Kafka["1"] = kafkaNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.Kafka = cv20.Notify.Kafka
|
||||
}
|
||||
if len(cv20.Notify.NATS) == 0 {
|
||||
srvConfig.Notify.NATS = make(map[string]natsNotify)
|
||||
srvConfig.Notify.NATS["1"] = natsNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.NATS = cv20.Notify.NATS
|
||||
}
|
||||
if len(cv20.Notify.Webhook) == 0 {
|
||||
srvConfig.Notify.Webhook = make(map[string]webhookNotify)
|
||||
srvConfig.Notify.Webhook["1"] = webhookNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.Webhook = cv20.Notify.Webhook
|
||||
}
|
||||
if len(cv20.Notify.MySQL) == 0 {
|
||||
srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
|
||||
srvConfig.Notify.MySQL["1"] = mySQLNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.MySQL = cv20.Notify.MySQL
|
||||
}
|
||||
if len(cv20.Notify.MQTT) == 0 {
|
||||
srvConfig.Notify.MQTT = make(map[string]mqttNotify)
|
||||
srvConfig.Notify.MQTT["1"] = mqttNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.MQTT = cv20.Notify.MQTT
|
||||
}
|
||||
|
||||
// Load browser config from existing config in the file.
|
||||
srvConfig.Browser = cv20.Browser
|
||||
|
||||
// Load domain config from existing config in the file.
|
||||
srvConfig.Domain = cv20.Domain
|
||||
|
||||
if err = quick.Save(configFile, srvConfig); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv20.Version, srvConfig.Version, err)
|
||||
}
|
||||
|
||||
log.Printf(configMigrateMSGTemplate, configFile, cv20.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
func migrateV21ToV22() error {
|
||||
configFile := getConfigFile()
|
||||
|
||||
cv21 := &serverConfigV21{}
|
||||
_, err := quick.Load(configFile, cv21)
|
||||
if os.IsNotExist(err) {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("Unable to load config version ‘21’. %v", err)
|
||||
}
|
||||
if cv21.Version != "21" {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy over fields from V21 into V22 config struct
|
||||
srvConfig := &serverConfigV22{
|
||||
Notify: notifier{},
|
||||
}
|
||||
srvConfig.Version = serverConfigVersion
|
||||
srvConfig.Credential = cv21.Credential
|
||||
srvConfig.Region = cv21.Region
|
||||
if srvConfig.Region == "" {
|
||||
// Region needs to be set for AWS Signature Version 4.
|
||||
srvConfig.Region = globalMinioDefaultRegion
|
||||
}
|
||||
|
||||
// check and set notifiers config
|
||||
if len(cv21.Notify.AMQP) == 0 {
|
||||
srvConfig.Notify.AMQP = make(map[string]amqpNotify)
|
||||
srvConfig.Notify.AMQP["1"] = amqpNotify{}
|
||||
} else {
|
||||
// New deliveryMode parameter is added for AMQP,
|
||||
// default value is already 0, so nothing to
|
||||
// explicitly migrate here.
|
||||
srvConfig.Notify.AMQP = cv21.Notify.AMQP
|
||||
}
|
||||
if len(cv21.Notify.ElasticSearch) == 0 {
|
||||
srvConfig.Notify.ElasticSearch = make(map[string]elasticSearchNotify)
|
||||
srvConfig.Notify.ElasticSearch["1"] = elasticSearchNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.ElasticSearch = cv21.Notify.ElasticSearch
|
||||
}
|
||||
if len(cv21.Notify.Redis) == 0 {
|
||||
srvConfig.Notify.Redis = make(map[string]redisNotify)
|
||||
srvConfig.Notify.Redis["1"] = redisNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.Redis = cv21.Notify.Redis
|
||||
}
|
||||
if len(cv21.Notify.PostgreSQL) == 0 {
|
||||
srvConfig.Notify.PostgreSQL = make(map[string]postgreSQLNotify)
|
||||
srvConfig.Notify.PostgreSQL["1"] = postgreSQLNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.PostgreSQL = cv21.Notify.PostgreSQL
|
||||
}
|
||||
if len(cv21.Notify.Kafka) == 0 {
|
||||
srvConfig.Notify.Kafka = make(map[string]kafkaNotify)
|
||||
srvConfig.Notify.Kafka["1"] = kafkaNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.Kafka = cv21.Notify.Kafka
|
||||
}
|
||||
if len(cv21.Notify.NATS) == 0 {
|
||||
srvConfig.Notify.NATS = make(map[string]natsNotify)
|
||||
srvConfig.Notify.NATS["1"] = natsNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.NATS = cv21.Notify.NATS
|
||||
}
|
||||
if len(cv21.Notify.Webhook) == 0 {
|
||||
srvConfig.Notify.Webhook = make(map[string]webhookNotify)
|
||||
srvConfig.Notify.Webhook["1"] = webhookNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.Webhook = cv21.Notify.Webhook
|
||||
}
|
||||
if len(cv21.Notify.MySQL) == 0 {
|
||||
srvConfig.Notify.MySQL = make(map[string]mySQLNotify)
|
||||
srvConfig.Notify.MySQL["1"] = mySQLNotify{
|
||||
Format: formatNamespace,
|
||||
}
|
||||
} else {
|
||||
srvConfig.Notify.MySQL = cv21.Notify.MySQL
|
||||
}
|
||||
if len(cv21.Notify.MQTT) == 0 {
|
||||
srvConfig.Notify.MQTT = make(map[string]mqttNotify)
|
||||
srvConfig.Notify.MQTT["1"] = mqttNotify{}
|
||||
} else {
|
||||
srvConfig.Notify.MQTT = cv21.Notify.MQTT
|
||||
}
|
||||
|
||||
// Load browser config from existing config in the file.
|
||||
srvConfig.Browser = cv21.Browser
|
||||
|
||||
// Load domain config from existing config in the file.
|
||||
srvConfig.Domain = cv21.Domain
|
||||
|
||||
if err = quick.Save(configFile, srvConfig); err != nil {
|
||||
return fmt.Errorf("Failed to migrate config from ‘%s’ to ‘%s’. %v", cv21.Version, srvConfig.Version, err)
|
||||
}
|
||||
|
||||
log.Printf(configMigrateMSGTemplate, configFile, cv21.Version, srvConfig.Version)
|
||||
return nil
|
||||
}
|
||||
|
||||
+21
-10
@@ -125,11 +125,16 @@ func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
if err := migrateV18ToV19(); err != nil {
|
||||
t.Fatal("migrate v18 to v19 should succeed when no config file is found")
|
||||
}
|
||||
|
||||
if err := migrateV19ToV20(); err != nil {
|
||||
t.Fatal("migrate v19 to v20 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV20ToV21(); err != nil {
|
||||
t.Fatal("migrate v20 to v21 should succeed when no config file is found")
|
||||
}
|
||||
}
|
||||
|
||||
// Test if a config migration from v2 to v19 is successfully done
|
||||
func TestServerConfigMigrateV2toV19(t *testing.T) {
|
||||
// Test if a config migration from v2 to v21 is successfully done
|
||||
func TestServerConfigMigrateV2toV21(t *testing.T) {
|
||||
rootPath, err := newTestConfig(globalMinioDefaultRegion)
|
||||
if err != nil {
|
||||
t.Fatalf("Init Test config failed")
|
||||
@@ -169,17 +174,17 @@ func TestServerConfigMigrateV2toV19(t *testing.T) {
|
||||
}
|
||||
|
||||
// Check the version number in the upgraded config file
|
||||
expectedVersion := v20
|
||||
if serverConfig.Version != expectedVersion {
|
||||
t.Fatalf("Expect version "+expectedVersion+", found: %v", serverConfig.Version)
|
||||
expectedVersion := serverConfigVersion
|
||||
if globalServerConfig.Version != expectedVersion {
|
||||
t.Fatalf("Expect version "+expectedVersion+", found: %v", globalServerConfig.Version)
|
||||
}
|
||||
|
||||
// Check if accessKey and secretKey are not altered during migration
|
||||
if serverConfig.Credential.AccessKey != accessKey {
|
||||
t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, serverConfig.Credential.AccessKey)
|
||||
if globalServerConfig.Credential.AccessKey != accessKey {
|
||||
t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, globalServerConfig.Credential.AccessKey)
|
||||
}
|
||||
if serverConfig.Credential.SecretKey != secretKey {
|
||||
t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, serverConfig.Credential.SecretKey)
|
||||
if globalServerConfig.Credential.SecretKey != secretKey {
|
||||
t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, globalServerConfig.Credential.SecretKey)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,6 +257,12 @@ func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
if err := migrateV18ToV19(); err == nil {
|
||||
t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV19ToV20(); err == nil {
|
||||
t.Fatal("migrateConfigV19ToV20() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV20ToV21(); err == nil {
|
||||
t.Fatal("migrateConfigV20ToV21() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
|
||||
@@ -418,6 +418,27 @@ type serverConfigV15 struct {
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// FileLogger is introduced to workaround the dependency about logrus
|
||||
type FileLogger struct {
|
||||
Enable bool `json:"enable"`
|
||||
Filename string `json:"filename"`
|
||||
}
|
||||
|
||||
// ConsoleLogger is introduced to workaround the dependency about logrus
|
||||
type ConsoleLogger struct {
|
||||
Enable bool `json:"enable"`
|
||||
}
|
||||
|
||||
// Loggers struct is defined with FileLogger and ConsoleLogger
|
||||
// although they are removed from logging logic. They are
|
||||
// kept here just to workaround the dependency migration
|
||||
// code/logic has on them.
|
||||
type loggers struct {
|
||||
sync.RWMutex
|
||||
Console ConsoleLogger `json:"console"`
|
||||
File FileLogger `json:"file"`
|
||||
}
|
||||
|
||||
// serverConfigV16 server configuration version '16' which is like
|
||||
// version '15' except it makes a change to logging configuration.
|
||||
type serverConfigV16 struct {
|
||||
@@ -490,3 +511,58 @@ type serverConfigV19 struct {
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV20 server configuration version '20' which is like
|
||||
// version '19' except it adds support for VirtualHostDomain
|
||||
type serverConfigV20 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV21 is just like version '20' without logger field
|
||||
type serverConfigV21 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifier `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV22 is just like version '21' with added support
|
||||
// for StorageClass.
|
||||
//
|
||||
// IMPORTANT NOTE: When updating this struct make sure that
|
||||
// serverConfig.ConfigDiff() is updated as necessary.
|
||||
type serverConfigV22 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser BrowserFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageClassConfig `json:"storageclass"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifier `json:"notify"`
|
||||
}
|
||||
@@ -1,74 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ConsoleLogger - console logger which logs into stderr.
|
||||
type ConsoleLogger struct {
|
||||
BaseLogTarget
|
||||
}
|
||||
|
||||
// Fire - log entry handler.
|
||||
func (logger ConsoleLogger) Fire(entry *logrus.Entry) error {
|
||||
if !logger.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
msgBytes, err := logger.formatter.Format(entry)
|
||||
if err == nil {
|
||||
fmt.Fprintf(os.Stderr, string(msgBytes))
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// String - represents ConsoleLogger as string.
|
||||
func (logger ConsoleLogger) String() string {
|
||||
enableStr := "disabled"
|
||||
if logger.Enable {
|
||||
enableStr = "enabled"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("console:%s", enableStr)
|
||||
}
|
||||
|
||||
// NewConsoleLogger - return new console logger object.
|
||||
func NewConsoleLogger() (logger ConsoleLogger) {
|
||||
logger.Enable = true
|
||||
logger.formatter = new(logrus.TextFormatter)
|
||||
|
||||
return logger
|
||||
}
|
||||
|
||||
// InitConsoleLogger - initializes console logger.
|
||||
func InitConsoleLogger(logger *ConsoleLogger) {
|
||||
if !logger.Enable {
|
||||
return
|
||||
}
|
||||
|
||||
if logger.formatter == nil {
|
||||
logger.formatter = new(logrus.TextFormatter)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
+12
-21
@@ -65,18 +65,9 @@ func (endpoint Endpoint) Type() EndpointType {
|
||||
return URLEndpointType
|
||||
}
|
||||
|
||||
// SetHTTPS - sets secure http for URLEndpointType.
|
||||
func (endpoint Endpoint) SetHTTPS() {
|
||||
if endpoint.Host != "" {
|
||||
endpoint.Scheme = "https"
|
||||
}
|
||||
}
|
||||
|
||||
// SetHTTP - sets insecure http for URLEndpointType.
|
||||
func (endpoint Endpoint) SetHTTP() {
|
||||
if endpoint.Host != "" {
|
||||
endpoint.Scheme = "http"
|
||||
}
|
||||
// IsHTTPS - returns true if secure for URLEndpointType.
|
||||
func (endpoint Endpoint) IsHTTPS() bool {
|
||||
return endpoint.Scheme == "https"
|
||||
}
|
||||
|
||||
// NewEndpoint - returns new endpoint based on given arguments.
|
||||
@@ -190,18 +181,18 @@ func (endpoints EndpointList) Less(i, j int) bool {
|
||||
return endpoints[i].String() < endpoints[j].String()
|
||||
}
|
||||
|
||||
// SetHTTPS - sets secure http for URLEndpointType.
|
||||
func (endpoints EndpointList) SetHTTPS() {
|
||||
for i := range endpoints {
|
||||
endpoints[i].SetHTTPS()
|
||||
}
|
||||
// IsHTTPS - returns true if secure for URLEndpointType.
|
||||
func (endpoints EndpointList) IsHTTPS() bool {
|
||||
return endpoints[0].IsHTTPS()
|
||||
}
|
||||
|
||||
// SetHTTP - sets insecure http for URLEndpointType.
|
||||
func (endpoints EndpointList) SetHTTP() {
|
||||
for i := range endpoints {
|
||||
endpoints[i].SetHTTP()
|
||||
// GetString - returns endpoint string of i-th endpoint (0-based),
|
||||
// and empty string for invalid indexes.
|
||||
func (endpoints EndpointList) GetString(i int) string {
|
||||
if i < 0 || i >= len(endpoints) {
|
||||
return ""
|
||||
}
|
||||
return endpoints[i].String()
|
||||
}
|
||||
|
||||
// NewEndpointList - returns new endpoint list based on input args.
|
||||
|
||||
@@ -19,6 +19,8 @@ package cmd
|
||||
import (
|
||||
"hash"
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// CreateFile creates a new bitrot encoded file spread over all available disks. CreateFile will create
|
||||
@@ -26,14 +28,14 @@ import (
|
||||
// be used to protect the erasure encoded file.
|
||||
func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer []byte, algorithm BitrotAlgorithm, writeQuorum int) (f ErasureFileInfo, err error) {
|
||||
if !algorithm.Available() {
|
||||
return f, traceError(errBitrotHashAlgoInvalid)
|
||||
return f, errors.Trace(errBitrotHashAlgoInvalid)
|
||||
}
|
||||
f.Checksums = make([][]byte, len(s.disks))
|
||||
hashers := make([]hash.Hash, len(s.disks))
|
||||
for i := range hashers {
|
||||
hashers[i] = algorithm.New()
|
||||
}
|
||||
errChans, errors := make([]chan error, len(s.disks)), make([]error, len(s.disks))
|
||||
errChans, errs := make([]chan error, len(s.disks)), make([]error, len(s.disks))
|
||||
for i := range errChans {
|
||||
errChans[i] = make(chan error, 1) // create buffered channel to let finished go-routines die early
|
||||
}
|
||||
@@ -53,19 +55,19 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
|
||||
return f, err
|
||||
}
|
||||
} else {
|
||||
return f, traceError(err)
|
||||
return f, errors.Trace(err)
|
||||
}
|
||||
|
||||
for i := range errChans { // span workers
|
||||
go erasureAppendFile(s.disks[i], volume, path, hashers[i], blocks[i], errChans[i])
|
||||
}
|
||||
for i := range errChans { // what until all workers are finished
|
||||
errors[i] = <-errChans[i]
|
||||
errs[i] = <-errChans[i]
|
||||
}
|
||||
if err = reduceWriteQuorumErrs(errors, objectOpIgnoredErrs, writeQuorum); err != nil {
|
||||
if err = reduceWriteQuorumErrs(errs, objectOpIgnoredErrs, writeQuorum); err != nil {
|
||||
return f, err
|
||||
}
|
||||
s.disks = evalDisks(s.disks, errors)
|
||||
s.disks = evalDisks(s.disks, errs)
|
||||
f.Size += int64(n)
|
||||
}
|
||||
|
||||
@@ -83,7 +85,7 @@ func (s *ErasureStorage) CreateFile(src io.Reader, volume, path string, buffer [
|
||||
// the hash of the written data. It sends the write error (or nil) over the error channel.
|
||||
func erasureAppendFile(disk StorageAPI, volume, path string, hash hash.Hash, buf []byte, errChan chan<- error) {
|
||||
if disk == OfflineDisk {
|
||||
errChan <- traceError(errDiskNotFound)
|
||||
errChan <- errors.Trace(errDiskNotFound)
|
||||
return
|
||||
}
|
||||
err := disk.AppendFile(volume, path, buf)
|
||||
|
||||
@@ -70,7 +70,7 @@ func TestErasureCreateFile(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to create test setup: %v", i, err)
|
||||
}
|
||||
storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
|
||||
storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
|
||||
if err != nil {
|
||||
setup.Remove()
|
||||
t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
|
||||
@@ -125,7 +125,7 @@ func benchmarkErasureWrite(data, parity, dataDown, parityDown int, size int64, b
|
||||
b.Fatalf("failed to create test setup: %v", err)
|
||||
}
|
||||
defer setup.Remove()
|
||||
storage, err := NewErasureStorage(setup.disks, data, parity)
|
||||
storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create ErasureStorage: %v", err)
|
||||
}
|
||||
|
||||
+16
-8
@@ -20,6 +20,8 @@ import (
|
||||
"fmt"
|
||||
"hash"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// HealFile tries to reconstruct an erasure-coded file spread over all
|
||||
@@ -48,7 +50,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
|
||||
f ErasureFileInfo, err error) {
|
||||
|
||||
if !alg.Available() {
|
||||
return f, traceError(errBitrotHashAlgoInvalid)
|
||||
return f, errors.Trace(errBitrotHashAlgoInvalid)
|
||||
}
|
||||
|
||||
// Initialization
|
||||
@@ -76,12 +78,14 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
|
||||
blocks[i] = make([]byte, chunksize)
|
||||
}
|
||||
var chunkOffset, blockOffset int64
|
||||
for ; blockOffset < size; blockOffset += blocksize {
|
||||
|
||||
// The for loop below is entered when size == 0 and
|
||||
// blockOffset == 0 to allow for reconstructing empty files.
|
||||
for ; blockOffset == 0 || blockOffset < size; blockOffset += blocksize {
|
||||
// last iteration may have less than blocksize data
|
||||
// left, so chunksize needs to be recomputed.
|
||||
if size < blockOffset+blocksize {
|
||||
blocksize = size - blockOffset
|
||||
chunksize = getChunkSize(blocksize, s.dataBlocks)
|
||||
chunksize = getChunkSize(size-blockOffset, s.dataBlocks)
|
||||
for i := range blocks {
|
||||
blocks[i] = blocks[i][:chunksize]
|
||||
}
|
||||
@@ -119,9 +123,13 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
|
||||
// iteration
|
||||
chunkOffset += chunksize
|
||||
|
||||
// reconstruct data - this computes all data and parity shards
|
||||
if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
|
||||
return f, err
|
||||
// reconstruct data - this computes all data and
|
||||
// parity shards - but we skip this step if we are
|
||||
// reconstructing an empty file.
|
||||
if chunksize > 0 {
|
||||
if err = s.ErasureDecodeDataAndParityBlocks(blocks); err != nil {
|
||||
return f, err
|
||||
}
|
||||
}
|
||||
|
||||
// write computed shards as chunks on file in each
|
||||
@@ -144,7 +152,7 @@ func (s ErasureStorage) HealFile(staleDisks []StorageAPI, volume, path string, b
|
||||
// If all disks had write errors we quit.
|
||||
if !writeSucceeded {
|
||||
// build error from all write errors
|
||||
return f, traceError(joinWriteErrors(writeErrors))
|
||||
return f, errors.Trace(joinWriteErrors(writeErrors))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -74,7 +74,7 @@ func TestErasureHealFile(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to setup XL environment: %v", i, err)
|
||||
}
|
||||
storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks)
|
||||
storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize)
|
||||
if err != nil {
|
||||
setup.Remove()
|
||||
t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
|
||||
|
||||
@@ -18,6 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// ReadFile reads as much data as requested from the file under the given volume and path and writes the data to the provided writer.
|
||||
@@ -25,13 +27,13 @@ import (
|
||||
// up to the given length. If parts of the file are corrupted ReadFile tries to reconstruct the data.
|
||||
func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset, length int64, totalLength int64, checksums [][]byte, algorithm BitrotAlgorithm, blocksize int64) (f ErasureFileInfo, err error) {
|
||||
if offset < 0 || length < 0 {
|
||||
return f, traceError(errUnexpected)
|
||||
return f, errors.Trace(errUnexpected)
|
||||
}
|
||||
if offset+length > totalLength {
|
||||
return f, traceError(errUnexpected)
|
||||
return f, errors.Trace(errUnexpected)
|
||||
}
|
||||
if !algorithm.Available() {
|
||||
return f, traceError(errBitrotHashAlgoInvalid)
|
||||
return f, errors.Trace(errBitrotHashAlgoInvalid)
|
||||
}
|
||||
|
||||
f.Checksums = make([][]byte, len(s.disks))
|
||||
@@ -66,7 +68,7 @@ func (s ErasureStorage) ReadFile(writer io.Writer, volume, path string, offset,
|
||||
}
|
||||
err = s.readConcurrent(volume, path, blockOffset, blocks, verifiers, errChans)
|
||||
if err != nil {
|
||||
return f, traceError(errXLReadQuorum)
|
||||
return f, errors.Trace(errXLReadQuorum)
|
||||
}
|
||||
|
||||
writeLength := blocksize - startOffset
|
||||
@@ -150,7 +152,7 @@ func erasureReadBlocksConcurrent(disks []StorageAPI, volume, path string, offset
|
||||
// It sends the returned error through the error channel.
|
||||
func erasureReadFromFile(disk StorageAPI, volume, path string, offset int64, buffer []byte, verifier *BitrotVerifier, errChan chan<- error) {
|
||||
if disk == OfflineDisk {
|
||||
errChan <- traceError(errDiskNotFound)
|
||||
errChan <- errors.Trace(errDiskNotFound)
|
||||
return
|
||||
}
|
||||
_, err := disk.ReadFile(volume, path, offset, buffer, verifier)
|
||||
|
||||
@@ -86,7 +86,7 @@ func TestErasureReadFile(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to create test setup: %v", i, err)
|
||||
}
|
||||
storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks)
|
||||
storage, err := NewErasureStorage(setup.disks, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize)
|
||||
if err != nil {
|
||||
setup.Remove()
|
||||
t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err)
|
||||
@@ -174,7 +174,7 @@ func TestErasureReadFileRandomOffsetLength(t *testing.T) {
|
||||
}
|
||||
defer setup.Remove()
|
||||
|
||||
storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks)
|
||||
storage, err := NewErasureStorage(setup.disks, dataBlocks, parityBlocks, blockSize)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create ErasureStorage: %v", err)
|
||||
}
|
||||
@@ -231,7 +231,7 @@ func benchmarkErasureRead(data, parity, dataDown, parityDown int, size int64, b
|
||||
b.Fatalf("failed to create test setup: %v", err)
|
||||
}
|
||||
defer setup.Remove()
|
||||
storage, err := NewErasureStorage(setup.disks, data, parity)
|
||||
storage, err := NewErasureStorage(setup.disks, data, parity, blockSizeV1)
|
||||
if err != nil {
|
||||
b.Fatalf("failed to create ErasureStorage: %v", err)
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"io"
|
||||
|
||||
"github.com/klauspost/reedsolomon"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// getDataBlockLen - get length of data blocks from encoded blocks.
|
||||
@@ -38,17 +39,17 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
|
||||
func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
|
||||
// Offset and out size cannot be negative.
|
||||
if offset < 0 || length < 0 {
|
||||
return 0, traceError(errUnexpected)
|
||||
return 0, errors.Trace(errUnexpected)
|
||||
}
|
||||
|
||||
// Do we have enough blocks?
|
||||
if len(enBlocks) < dataBlocks {
|
||||
return 0, traceError(reedsolomon.ErrTooFewShards)
|
||||
return 0, errors.Trace(reedsolomon.ErrTooFewShards)
|
||||
}
|
||||
|
||||
// Do we have enough data?
|
||||
if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
|
||||
return 0, traceError(reedsolomon.ErrShortData)
|
||||
return 0, errors.Trace(reedsolomon.ErrShortData)
|
||||
}
|
||||
|
||||
// Counter to decrement total left to write.
|
||||
@@ -76,7 +77,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
|
||||
if write < int64(len(block)) {
|
||||
n, err := io.Copy(dst, bytes.NewReader(block[:write]))
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
totalWritten += n
|
||||
break
|
||||
@@ -84,7 +85,7 @@ func writeDataBlocks(dst io.Writer, enBlocks [][]byte, dataBlocks int, offset in
|
||||
// Copy the block.
|
||||
n, err := io.Copy(dst, bytes.NewReader(block))
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
// Decrement output size.
|
||||
|
||||
+9
-7
@@ -21,6 +21,7 @@ import (
|
||||
"hash"
|
||||
|
||||
"github.com/klauspost/reedsolomon"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// OfflineDisk represents an unavailable disk.
|
||||
@@ -43,10 +44,11 @@ type ErasureStorage struct {
|
||||
|
||||
// NewErasureStorage creates a new ErasureStorage. The storage erasure codes and protects all data written to
|
||||
// the disks.
|
||||
func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int) (s ErasureStorage, err error) {
|
||||
erasure, err := reedsolomon.New(dataBlocks, parityBlocks)
|
||||
func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int, blockSize int64) (s ErasureStorage, err error) {
|
||||
shardsize := (int(blockSize) + dataBlocks - 1) / dataBlocks
|
||||
erasure, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(shardsize))
|
||||
if err != nil {
|
||||
return s, traceErrorf("failed to create erasure coding: %v", err)
|
||||
return s, errors.Tracef("failed to create erasure coding: %v", err)
|
||||
}
|
||||
s = ErasureStorage{
|
||||
disks: make([]StorageAPI, len(disks)),
|
||||
@@ -63,10 +65,10 @@ func NewErasureStorage(disks []StorageAPI, dataBlocks, parityBlocks int) (s Eras
|
||||
func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
|
||||
encoded, err := s.erasure.Split(data)
|
||||
if err != nil {
|
||||
return nil, traceErrorf("failed to split data: %v", err)
|
||||
return nil, errors.Tracef("failed to split data: %v", err)
|
||||
}
|
||||
if err = s.erasure.Encode(encoded); err != nil {
|
||||
return nil, traceErrorf("failed to encode data: %v", err)
|
||||
return nil, errors.Tracef("failed to encode data: %v", err)
|
||||
}
|
||||
return encoded, nil
|
||||
}
|
||||
@@ -76,7 +78,7 @@ func (s *ErasureStorage) ErasureEncode(data []byte) ([][]byte, error) {
|
||||
// It returns an error if the decoding failed.
|
||||
func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
|
||||
if err := s.erasure.ReconstructData(data); err != nil {
|
||||
return traceErrorf("failed to reconstruct data: %v", err)
|
||||
return errors.Tracef("failed to reconstruct data: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -85,7 +87,7 @@ func (s *ErasureStorage) ErasureDecodeDataBlocks(data [][]byte) error {
|
||||
// It returns an error if the decoding failed.
|
||||
func (s *ErasureStorage) ErasureDecodeDataAndParityBlocks(data [][]byte) error {
|
||||
if err := s.erasure.Reconstruct(data); err != nil {
|
||||
return traceErrorf("failed to reconstruct data: %v", err)
|
||||
return errors.Tracef("failed to reconstruct data: %v", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+1
-1
@@ -52,7 +52,7 @@ func TestErasureDecode(t *testing.T) {
|
||||
copy(buffer, data)
|
||||
|
||||
disks := make([]StorageAPI, test.dataBlocks+test.parityBlocks)
|
||||
storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks)
|
||||
storage, err := NewErasureStorage(disks, test.dataBlocks, test.parityBlocks, blockSizeV1)
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: failed to create erasure storage: %v", i, err)
|
||||
}
|
||||
|
||||
-158
@@ -1,158 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Holds the current directory path. Used for trimming path in traceError()
|
||||
var rootPath string
|
||||
|
||||
// Figure out the rootPath
|
||||
func initError() {
|
||||
// Root path is automatically determined from the calling function's source file location.
|
||||
// Catch the calling function's source file path.
|
||||
_, file, _, _ := runtime.Caller(1)
|
||||
// Save the directory alone.
|
||||
rootPath = filepath.Dir(file)
|
||||
}
|
||||
|
||||
// Represents a stack frame in the stack trace.
|
||||
type traceInfo struct {
|
||||
file string // File where error occurred
|
||||
line int // Line where error occurred
|
||||
name string // Name of the function where error occurred
|
||||
}
|
||||
|
||||
// Error - error type containing cause and the stack trace.
|
||||
type Error struct {
|
||||
e error // Holds the cause error
|
||||
trace []traceInfo // stack trace
|
||||
errs []error // Useful for XL to hold errors from all disks
|
||||
}
|
||||
|
||||
// Implement error interface.
|
||||
func (e Error) Error() string {
|
||||
return e.e.Error()
|
||||
}
|
||||
|
||||
// Trace - returns stack trace.
|
||||
func (e Error) Trace() []string {
|
||||
var traceArr []string
|
||||
for _, info := range e.trace {
|
||||
traceArr = append(traceArr, fmt.Sprintf("%s:%d:%s",
|
||||
info.file, info.line, info.name))
|
||||
}
|
||||
return traceArr
|
||||
}
|
||||
|
||||
// NewStorageError - return new Error type.
|
||||
func traceError(e error, errs ...error) error {
|
||||
if e == nil {
|
||||
return nil
|
||||
}
|
||||
err := &Error{}
|
||||
err.e = e
|
||||
err.errs = errs
|
||||
|
||||
stack := make([]uintptr, 40)
|
||||
length := runtime.Callers(2, stack)
|
||||
if length > len(stack) {
|
||||
length = len(stack)
|
||||
}
|
||||
stack = stack[:length]
|
||||
|
||||
for _, pc := range stack {
|
||||
pc = pc - 1
|
||||
fn := runtime.FuncForPC(pc)
|
||||
file, line := fn.FileLine(pc)
|
||||
name := fn.Name()
|
||||
if hasSuffix(name, "ServeHTTP") {
|
||||
break
|
||||
}
|
||||
if hasSuffix(name, "runtime.") {
|
||||
break
|
||||
}
|
||||
|
||||
file = strings.TrimPrefix(file, rootPath+string(os.PathSeparator))
|
||||
name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.")
|
||||
err.trace = append(err.trace, traceInfo{file, line, name})
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Returns the underlying cause error.
|
||||
func errorCause(err error) error {
|
||||
if e, ok := err.(*Error); ok {
|
||||
err = e.e
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Returns slice of underlying cause error.
|
||||
func errorsCause(errs []error) []error {
|
||||
cerrs := make([]error, len(errs))
|
||||
for i, err := range errs {
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
cerrs[i] = errorCause(err)
|
||||
}
|
||||
return cerrs
|
||||
}
|
||||
|
||||
// Collection of basic errors.
|
||||
var baseErrs = []error{
|
||||
errDiskNotFound,
|
||||
errFaultyDisk,
|
||||
errFaultyRemoteDisk,
|
||||
}
|
||||
|
||||
var baseIgnoredErrs = baseErrs
|
||||
|
||||
// isErrIgnored returns whether given error is ignored or not.
|
||||
func isErrIgnored(err error, ignoredErrs ...error) bool {
|
||||
err = errorCause(err)
|
||||
for _, ignoredErr := range ignoredErrs {
|
||||
if ignoredErr == err {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isErr returns whether given error is exact error.
|
||||
func isErr(err error, errs ...error) bool {
|
||||
err = errorCause(err)
|
||||
for _, exactErr := range errs {
|
||||
if err == exactErr {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// traceErrorf behaves like fmt.traceErrorf but also traces the returned error.
|
||||
func traceErrorf(format string, args ...interface{}) error {
|
||||
return traceError(fmt.Errorf(format, args...))
|
||||
}
|
||||
+23
-60
@@ -27,6 +27,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
@@ -106,10 +107,10 @@ func newNotificationEvent(event eventData) NotificationEvent {
|
||||
}
|
||||
|
||||
// Fetch the region.
|
||||
region := serverConfig.GetRegion()
|
||||
region := globalServerConfig.GetRegion()
|
||||
|
||||
// Fetch the credentials.
|
||||
creds := serverConfig.GetCredential()
|
||||
creds := globalServerConfig.GetCredential()
|
||||
|
||||
// Time when Minio finished processing the request.
|
||||
eventTime := UTCNow()
|
||||
@@ -364,21 +365,14 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
|
||||
// Construct the notification config path.
|
||||
ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
|
||||
|
||||
// Acquire a write lock on notification config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
|
||||
if err := objLock.GetRLock(globalOperationTimeout); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer objLock.RUnlock()
|
||||
|
||||
var buffer bytes.Buffer
|
||||
err := objAPI.GetObject(minioMetaBucket, ncPath, 0, -1, &buffer) // Read everything.
|
||||
err := objAPI.GetObject(minioMetaBucket, ncPath, 0, -1, &buffer, "") // Read everything.
|
||||
if err != nil {
|
||||
// 'notification.xml' not found return
|
||||
// 'errNoSuchNotifications'. This is default when no
|
||||
// bucket notifications are found on the bucket.
|
||||
if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
return nil, errors.Trace(errNoSuchNotifications)
|
||||
}
|
||||
errorIf(err, "Unable to load bucket-notification for bucket %s", bucket)
|
||||
// Returns error for other errors.
|
||||
@@ -387,7 +381,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
|
||||
|
||||
// if `notifications.xml` is empty we should return NoSuchNotifications.
|
||||
if buffer.Len() == 0 {
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
return nil, errors.Trace(errNoSuchNotifications)
|
||||
}
|
||||
|
||||
// Unmarshal notification bytes.
|
||||
@@ -395,7 +389,7 @@ func loadNotificationConfig(bucket string, objAPI ObjectLayer) (*notificationCon
|
||||
notificationCfg := ¬ificationConfig{}
|
||||
// Unmarshal notification bytes only if we read data.
|
||||
if err = xml.Unmarshal(notificationConfigBytes, notificationCfg); err != nil {
|
||||
return nil, traceError(err)
|
||||
return nil, errors.Trace(err)
|
||||
}
|
||||
|
||||
// Return success.
|
||||
@@ -415,21 +409,14 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
|
||||
// Construct the notification config path.
|
||||
lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
|
||||
|
||||
// Acquire a write lock on notification config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
|
||||
if err := objLock.GetRLock(globalOperationTimeout); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer objLock.RUnlock()
|
||||
|
||||
var buffer bytes.Buffer
|
||||
err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer)
|
||||
err := objAPI.GetObject(minioMetaBucket, lcPath, 0, -1, &buffer, "")
|
||||
if err != nil {
|
||||
// 'listener.json' not found return
|
||||
// 'errNoSuchNotifications'. This is default when no
|
||||
// bucket listeners are found on the bucket
|
||||
if isErrObjectNotFound(err) || isErrIncompleteBody(err) {
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
return nil, errors.Trace(errNoSuchNotifications)
|
||||
}
|
||||
errorIf(err, "Unable to load bucket-listeners for bucket %s", bucket)
|
||||
// Returns error for other errors.
|
||||
@@ -438,14 +425,14 @@ func loadListenerConfig(bucket string, objAPI ObjectLayer) ([]listenerConfig, er
|
||||
|
||||
// if `listener.json` is empty we should return NoSuchNotifications.
|
||||
if buffer.Len() == 0 {
|
||||
return nil, traceError(errNoSuchNotifications)
|
||||
return nil, errors.Trace(errNoSuchNotifications)
|
||||
}
|
||||
|
||||
var lCfg []listenerConfig
|
||||
lConfigBytes := buffer.Bytes()
|
||||
if err = json.Unmarshal(lConfigBytes, &lCfg); err != nil {
|
||||
errorIf(err, "Unable to unmarshal listener config from JSON.")
|
||||
return nil, traceError(err)
|
||||
return nil, errors.Trace(err)
|
||||
}
|
||||
|
||||
// Return success.
|
||||
@@ -462,12 +449,6 @@ func persistNotificationConfig(bucket string, ncfg *notificationConfig, obj Obje
|
||||
|
||||
// build path
|
||||
ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
|
||||
// Acquire a write lock on notification config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
// write object to path
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
|
||||
@@ -493,12 +474,6 @@ func persistListenerConfig(bucket string, lcfg []listenerConfig, obj ObjectLayer
|
||||
|
||||
// build path
|
||||
lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
|
||||
// Acquire a write lock on notification config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
|
||||
if err = objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
|
||||
// write object to path
|
||||
hashReader, err := hash.NewReader(bytes.NewReader(buf), int64(len(buf)), "", getSHA256Hash(buf))
|
||||
@@ -525,12 +500,6 @@ func removeNotificationConfig(bucket string, objAPI ObjectLayer) error {
|
||||
|
||||
ncPath := path.Join(bucketConfigPrefix, bucket, bucketNotificationConfig)
|
||||
|
||||
// Acquire a write lock on notification config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, ncPath)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
return objAPI.DeleteObject(minioMetaBucket, ncPath)
|
||||
}
|
||||
|
||||
@@ -539,12 +508,6 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
|
||||
// make the path
|
||||
lcPath := path.Join(bucketConfigPrefix, bucket, bucketListenerConfig)
|
||||
|
||||
// Acquire a write lock on notification config before modifying.
|
||||
objLock := globalNSMutex.NewNSLock(minioMetaBucket, lcPath)
|
||||
if err := objLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objLock.Unlock()
|
||||
return objAPI.DeleteObject(minioMetaBucket, lcPath)
|
||||
}
|
||||
|
||||
@@ -552,13 +515,13 @@ func removeListenerConfig(bucket string, objAPI ObjectLayer) error {
|
||||
func loadNotificationAndListenerConfig(bucketName string, objAPI ObjectLayer) (nCfg *notificationConfig, lCfg []listenerConfig, err error) {
|
||||
// Loads notification config if any.
|
||||
nCfg, err = loadNotificationConfig(bucketName, objAPI)
|
||||
if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
|
||||
if err != nil && !errors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Loads listener config if any.
|
||||
lCfg, err = loadListenerConfig(bucketName, objAPI)
|
||||
if err != nil && !isErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
|
||||
if err != nil && !errors.IsErrIgnored(err, errDiskNotFound, errNoSuchNotifications) {
|
||||
return nil, nil, err
|
||||
}
|
||||
return nCfg, lCfg, nil
|
||||
@@ -595,7 +558,7 @@ func addQueueTarget(queueTargets map[string]*logrus.Logger,
|
||||
newTargetFunc func(string) (*logrus.Logger, error)) (string, error) {
|
||||
|
||||
// Construct the queue ARN for AMQP.
|
||||
queueARN := minioSqs + serverConfig.GetRegion() + ":" + accountID + ":" + queueType
|
||||
queueARN := minioSqs + globalServerConfig.GetRegion() + ":" + accountID + ":" + queueType
|
||||
|
||||
// Queue target if already initialized we move to the next ARN.
|
||||
if _, ok := queueTargets[queueARN]; ok {
|
||||
@@ -618,7 +581,7 @@ func addQueueTarget(queueTargets map[string]*logrus.Logger,
|
||||
func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
queueTargets := make(map[string]*logrus.Logger)
|
||||
// Load all amqp targets, initialize their respective loggers.
|
||||
for accountID, amqpN := range serverConfig.Notify.GetAMQP() {
|
||||
for accountID, amqpN := range globalServerConfig.Notify.GetAMQP() {
|
||||
if !amqpN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -637,7 +600,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load all mqtt targets, initialize their respective loggers.
|
||||
for accountID, mqttN := range serverConfig.Notify.GetMQTT() {
|
||||
for accountID, mqttN := range globalServerConfig.Notify.GetMQTT() {
|
||||
if !mqttN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -656,7 +619,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load all nats targets, initialize their respective loggers.
|
||||
for accountID, natsN := range serverConfig.Notify.GetNATS() {
|
||||
for accountID, natsN := range globalServerConfig.Notify.GetNATS() {
|
||||
if !natsN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -675,7 +638,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load redis targets, initialize their respective loggers.
|
||||
for accountID, redisN := range serverConfig.Notify.GetRedis() {
|
||||
for accountID, redisN := range globalServerConfig.Notify.GetRedis() {
|
||||
if !redisN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -694,7 +657,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load Webhook targets, initialize their respective loggers.
|
||||
for accountID, webhookN := range serverConfig.Notify.GetWebhook() {
|
||||
for accountID, webhookN := range globalServerConfig.Notify.GetWebhook() {
|
||||
if !webhookN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -704,7 +667,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load elastic targets, initialize their respective loggers.
|
||||
for accountID, elasticN := range serverConfig.Notify.GetElasticSearch() {
|
||||
for accountID, elasticN := range globalServerConfig.Notify.GetElasticSearch() {
|
||||
if !elasticN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -723,7 +686,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load PostgreSQL targets, initialize their respective loggers.
|
||||
for accountID, pgN := range serverConfig.Notify.GetPostgreSQL() {
|
||||
for accountID, pgN := range globalServerConfig.Notify.GetPostgreSQL() {
|
||||
if !pgN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -742,7 +705,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load MySQL targets, initialize their respective loggers.
|
||||
for accountID, msqlN := range serverConfig.Notify.GetMySQL() {
|
||||
for accountID, msqlN := range globalServerConfig.Notify.GetMySQL() {
|
||||
if !msqlN.Enable {
|
||||
continue
|
||||
}
|
||||
@@ -761,7 +724,7 @@ func loadAllQueueTargets() (map[string]*logrus.Logger, error) {
|
||||
}
|
||||
|
||||
// Load Kafka targets, initialize their respective loggers.
|
||||
for accountID, kafkaN := range serverConfig.Notify.GetKafka() {
|
||||
for accountID, kafkaN := range globalServerConfig.Notify.GetKafka() {
|
||||
if !kafkaN.Enable {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -23,6 +23,8 @@ import (
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// Test InitEventNotifier with faulty disks
|
||||
@@ -71,7 +73,7 @@ func TestInitEventNotifierFaultyDisks(t *testing.T) {
|
||||
}
|
||||
// Test initEventNotifier() with faulty disks
|
||||
for i := 1; i <= 3; i++ {
|
||||
if err := initEventNotifier(xl); errorCause(err) != errFaultyDisk {
|
||||
if err := initEventNotifier(xl); errors.Cause(err) != errFaultyDisk {
|
||||
t.Fatal("Unexpected error:", err)
|
||||
}
|
||||
}
|
||||
@@ -98,7 +100,7 @@ func TestInitEventNotifierWithPostgreSQL(t *testing.T) {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
serverConfig.Notify.SetPostgreSQLByID("1", postgreSQLNotify{Enable: true})
|
||||
globalServerConfig.Notify.SetPostgreSQLByID("1", postgreSQLNotify{Enable: true})
|
||||
if err := initEventNotifier(fs); err == nil {
|
||||
t.Fatal("PostgreSQL config didn't fail.")
|
||||
}
|
||||
@@ -125,7 +127,7 @@ func TestInitEventNotifierWithNATS(t *testing.T) {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
serverConfig.Notify.SetNATSByID("1", natsNotify{Enable: true})
|
||||
globalServerConfig.Notify.SetNATSByID("1", natsNotify{Enable: true})
|
||||
if err := initEventNotifier(fs); err == nil {
|
||||
t.Fatal("NATS config didn't fail.")
|
||||
}
|
||||
@@ -152,7 +154,7 @@ func TestInitEventNotifierWithWebHook(t *testing.T) {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
serverConfig.Notify.SetWebhookByID("1", webhookNotify{Enable: true})
|
||||
globalServerConfig.Notify.SetWebhookByID("1", webhookNotify{Enable: true})
|
||||
if err := initEventNotifier(fs); err == nil {
|
||||
t.Fatal("WebHook config didn't fail.")
|
||||
}
|
||||
@@ -179,7 +181,7 @@ func TestInitEventNotifierWithAMQP(t *testing.T) {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
serverConfig.Notify.SetAMQPByID("1", amqpNotify{Enable: true})
|
||||
globalServerConfig.Notify.SetAMQPByID("1", amqpNotify{Enable: true})
|
||||
if err := initEventNotifier(fs); err == nil {
|
||||
t.Fatal("AMQP config didn't fail.")
|
||||
}
|
||||
@@ -206,7 +208,7 @@ func TestInitEventNotifierWithElasticSearch(t *testing.T) {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
serverConfig.Notify.SetElasticSearchByID("1", elasticSearchNotify{Enable: true})
|
||||
globalServerConfig.Notify.SetElasticSearchByID("1", elasticSearchNotify{Enable: true})
|
||||
if err := initEventNotifier(fs); err == nil {
|
||||
t.Fatal("ElasticSearch config didn't fail.")
|
||||
}
|
||||
@@ -233,7 +235,7 @@ func TestInitEventNotifierWithRedis(t *testing.T) {
|
||||
t.Fatal("Unable to initialize FS backend.", err)
|
||||
}
|
||||
|
||||
serverConfig.Notify.SetRedisByID("1", redisNotify{Enable: true})
|
||||
globalServerConfig.Notify.SetRedisByID("1", redisNotify{Enable: true})
|
||||
if err := initEventNotifier(fs); err == nil {
|
||||
t.Fatal("Redis config didn't fail.")
|
||||
}
|
||||
@@ -415,7 +417,7 @@ func TestListenBucketNotification(t *testing.T) {
|
||||
|
||||
listenARN := fmt.Sprintf("%s:%s:1:%s-%s",
|
||||
minioTopic,
|
||||
serverConfig.GetRegion(),
|
||||
globalServerConfig.GetRegion(),
|
||||
snsTypeMinio,
|
||||
s.testServer.Server.Listener.Addr(),
|
||||
)
|
||||
@@ -523,7 +525,7 @@ func TestAddRemoveBucketListenerConfig(t *testing.T) {
|
||||
accountID := fmt.Sprintf("%d", UTCNow().UnixNano())
|
||||
accountARN := fmt.Sprintf(
|
||||
"arn:minio:sqs:%s:%s:listen-%s",
|
||||
serverConfig.GetRegion(),
|
||||
globalServerConfig.GetRegion(),
|
||||
accountID,
|
||||
globalMinioAddr,
|
||||
)
|
||||
|
||||
@@ -1,86 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/Sirupsen/logrus"
|
||||
)
|
||||
|
||||
// FileLogger - file logger which logs to a file.
|
||||
type FileLogger struct {
|
||||
BaseLogTarget
|
||||
Filename string `json:"filename"`
|
||||
file *os.File
|
||||
}
|
||||
|
||||
// Fire - log entry handler.
|
||||
func (logger FileLogger) Fire(entry *logrus.Entry) (err error) {
|
||||
if !logger.Enable {
|
||||
return nil
|
||||
}
|
||||
|
||||
msgBytes, err := logger.formatter.Format(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if _, err = logger.file.Write(msgBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = logger.file.Sync()
|
||||
return err
|
||||
}
|
||||
|
||||
// String - represents ConsoleLogger as string.
|
||||
func (logger FileLogger) String() string {
|
||||
enableStr := "disabled"
|
||||
if logger.Enable {
|
||||
enableStr = "enabled"
|
||||
}
|
||||
|
||||
return fmt.Sprintf("file:%s:%s", enableStr, logger.Filename)
|
||||
}
|
||||
|
||||
// NewFileLogger - creates new file logger object.
|
||||
func NewFileLogger(filename string) (logger FileLogger) {
|
||||
logger.Enable = true
|
||||
logger.formatter = new(logrus.JSONFormatter)
|
||||
logger.Filename = filename
|
||||
|
||||
return logger
|
||||
}
|
||||
|
||||
// InitFileLogger - initializes file logger.
|
||||
func InitFileLogger(logger *FileLogger) (err error) {
|
||||
if !logger.Enable {
|
||||
return err
|
||||
}
|
||||
|
||||
if logger.formatter == nil {
|
||||
logger.formatter = new(logrus.JSONFormatter)
|
||||
}
|
||||
|
||||
if logger.file == nil {
|
||||
logger.file, err = os.OpenFile(logger.Filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0664)
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
errors2 "github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// FS format version strings.
|
||||
const (
|
||||
formatBackendFS = "fs"
|
||||
formatFSVersionV1 = "1"
|
||||
formatFSVersionV2 = "2"
|
||||
)
|
||||
|
||||
// formatFSV1 - structure holds format version '1'.
|
||||
type formatFSV1 struct {
|
||||
formatMetaV1
|
||||
FS struct {
|
||||
Version string `json:"version"`
|
||||
} `json:"fs"`
|
||||
}
|
||||
|
||||
// formatFSV2 - structure is same as formatFSV1. But the multipart backend
|
||||
// structure is flat instead of hierarchy now.
|
||||
// In .minio.sys/multipart we have:
|
||||
// sha256(bucket/object)/uploadID/[fs.json, 1.etag, 2.etag ....]
|
||||
type formatFSV2 = formatFSV1
|
||||
|
||||
// Used to detect the version of "fs" format.
|
||||
type formatFSVersionDetect struct {
|
||||
FS struct {
|
||||
Version string `json:"version"`
|
||||
} `json:"fs"`
|
||||
}
|
||||
|
||||
// Returns the latest "fs" format V1
|
||||
func newFormatFSV1() (format *formatFSV1) {
|
||||
f := &formatFSV1{}
|
||||
f.Version = formatMetaVersionV1
|
||||
f.Format = formatBackendFS
|
||||
f.FS.Version = formatFSVersionV1
|
||||
return f
|
||||
}
|
||||
|
||||
// Returns the latest "fs" format V2
|
||||
func newFormatFSV2() (format *formatFSV2) {
|
||||
f := &formatFSV2{}
|
||||
f.Version = formatMetaVersionV1
|
||||
f.Format = formatBackendFS
|
||||
f.FS.Version = formatFSVersionV2
|
||||
return f
|
||||
}
|
||||
|
||||
// Returns the field formatMetaV1.Format i.e the string "fs" which is never likely to change.
|
||||
// We do not use this function in XL to get the format as the file is not fcntl-locked on XL.
|
||||
func formatMetaGetFormatBackendFS(r io.ReadSeeker) (string, error) {
|
||||
format := &formatMetaV1{}
|
||||
if err := jsonLoad(r, format); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if format.Version == formatMetaVersionV1 {
|
||||
return format.Format, nil
|
||||
}
|
||||
return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, format.Version)
|
||||
}
|
||||
|
||||
// Returns formatFS.FS.Version
|
||||
func formatFSGetVersion(r io.ReadSeeker) (string, error) {
|
||||
format := &formatFSVersionDetect{}
|
||||
if err := jsonLoad(r, format); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return format.FS.Version, nil
|
||||
}
|
||||
|
||||
// Migrate from V1 to V2. V2 implements new backend format for multipart
|
||||
// uploads. Delete the previous multipart directory.
|
||||
func formatFSMigrateV1ToV2(wlk *lock.LockedFile, fsPath string) error {
|
||||
version, err := formatFSGetVersion(wlk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if version != formatFSVersionV1 {
|
||||
return fmt.Errorf(`format.json version expected %s, found %s`, formatFSVersionV1, version)
|
||||
}
|
||||
|
||||
if err = fsRemoveAll(path.Join(fsPath, minioMetaMultipartBucket)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(path.Join(fsPath, minioMetaMultipartBucket), 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return jsonSave(wlk.File, newFormatFSV2())
|
||||
}
|
||||
|
||||
// Migrate the "fs" backend.
|
||||
// Migration should happen when formatFSV1.FS.Version changes. This version
|
||||
// can change when there is a change to the struct formatFSV1.FS or if there
|
||||
// is any change in the backend file system tree structure.
|
||||
func formatFSMigrate(wlk *lock.LockedFile, fsPath string) error {
|
||||
// Add any migration code here in case we bump format.FS.Version
|
||||
version, err := formatFSGetVersion(wlk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
switch version {
|
||||
case formatFSVersionV1:
|
||||
if err = formatFSMigrateV1ToV2(wlk, fsPath); err != nil {
|
||||
return err
|
||||
}
|
||||
fallthrough
|
||||
case formatFSVersionV2:
|
||||
// We are at the latest version.
|
||||
}
|
||||
|
||||
// Make sure that the version is what we expect after the migration.
|
||||
version, err = formatFSGetVersion(wlk)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if version != formatFSVersionV2 {
|
||||
return fmt.Errorf(`%s file: expected FS version: %s, found FS version: %s`, formatConfigFile, formatFSVersionV2, version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Creates a new format.json if unformatted.
|
||||
func createFormatFS(fsFormatPath string) error {
|
||||
// Attempt a write lock on formatConfigFile `format.json`
|
||||
// file stored in minioMetaBucket(.minio.sys) directory.
|
||||
lk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return errors2.Trace(err)
|
||||
}
|
||||
// Close the locked file upon return.
|
||||
defer lk.Close()
|
||||
|
||||
fi, err := lk.Stat()
|
||||
if err != nil {
|
||||
return errors2.Trace(err)
|
||||
}
|
||||
if fi.Size() != 0 {
|
||||
// format.json already got created because of another minio process's createFormatFS()
|
||||
return nil
|
||||
}
|
||||
|
||||
return jsonSave(lk.File, newFormatFSV1())
|
||||
}
|
||||
|
||||
// This function returns a read-locked format.json reference to the caller.
|
||||
// The file descriptor should be kept open throughout the life
|
||||
// of the process so that another minio process does not try to
|
||||
// migrate the backend when we are actively working on the backend.
|
||||
func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
|
||||
fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
|
||||
// Any read on format.json should be done with read-lock.
|
||||
// Any write on format.json should be done with write-lock.
|
||||
for {
|
||||
isEmpty := false
|
||||
rlk, err := lock.RLockedOpenFile(fsFormatPath)
|
||||
if err == nil {
|
||||
// format.json can be empty in a rare condition when another
|
||||
// minio process just created the file but could not hold lock
|
||||
// and write to it.
|
||||
var fi os.FileInfo
|
||||
fi, err = rlk.Stat()
|
||||
if err != nil {
|
||||
return nil, errors2.Trace(err)
|
||||
}
|
||||
isEmpty = fi.Size() == 0
|
||||
}
|
||||
if os.IsNotExist(err) || isEmpty {
|
||||
if err == nil {
|
||||
rlk.Close()
|
||||
}
|
||||
// Fresh disk - create format.json
|
||||
err = createFormatFS(fsFormatPath)
|
||||
if err == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again.
|
||||
// Can happen in a rare situation when a parallel minio process
|
||||
// holds the lock and creates format.json
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors2.Trace(err)
|
||||
}
|
||||
// After successfully creating format.json try to hold a read-lock on
|
||||
// the file.
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, errors2.Trace(err)
|
||||
}
|
||||
|
||||
formatBackend, err := formatMetaGetFormatBackendFS(rlk)
|
||||
if err != nil {
|
||||
return nil, errors2.Trace(err)
|
||||
}
|
||||
if formatBackend != formatBackendFS {
|
||||
return nil, fmt.Errorf(`%s file: expected format-type: %s, found: %s`, formatConfigFile, formatBackendFS, formatBackend)
|
||||
}
|
||||
version, err := formatFSGetVersion(rlk)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if version != formatFSVersionV2 {
|
||||
// Format needs migration
|
||||
rlk.Close()
|
||||
// Hold write lock during migration so that we do not disturb any
|
||||
// minio processes running in parallel.
|
||||
wlk, err := lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
|
||||
if err == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again.
|
||||
wlk.Close()
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = formatFSMigrate(wlk, fsPath)
|
||||
wlk.Close()
|
||||
if err != nil {
|
||||
// Migration failed, bail out so that the user can observe what happened.
|
||||
return nil, err
|
||||
}
|
||||
// Successfully migrated, now try to hold a read-lock on format.json
|
||||
continue
|
||||
}
|
||||
|
||||
return rlk, nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestFSFormatFS - tests initFormatFS, formatMetaGetFormatBackendFS, formatFSGetVersion.
|
||||
func TestFSFormatFS(t *testing.T) {
|
||||
// Prepare for testing
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer os.RemoveAll(disk)
|
||||
|
||||
fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
|
||||
|
||||
// Assign a new UUID.
|
||||
uuid := mustGetUUID()
|
||||
|
||||
// Initialize meta volume, if volume already exists ignores it.
|
||||
if err := initMetaVolumeFS(disk, uuid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
rlk, err := initFormatFS(disk)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rlk.Close()
|
||||
|
||||
// Do the basic sanity checks to check if initFormatFS() did its job.
|
||||
f, err := os.OpenFile(fsFormatPath, os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
format, err := formatMetaGetFormatBackendFS(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if format != formatBackendFS {
|
||||
t.Fatalf(`expected: %s, got: %s`, formatBackendFS, format)
|
||||
}
|
||||
version, err := formatFSGetVersion(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != formatFSVersionV2 {
|
||||
t.Fatalf(`expected: %s, got: %s`, formatFSVersionV2, version)
|
||||
}
|
||||
|
||||
// Corrupt the format.json file and test the functions.
|
||||
// formatMetaGetFormatBackendFS, formatFSGetVersion, initFormatFS should return errors.
|
||||
if err = f.Truncate(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = f.WriteString("b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, err = formatMetaGetFormatBackendFS(f); err == nil {
|
||||
t.Fatal("expected to fail")
|
||||
}
|
||||
if _, err = formatFSGetVersion(rlk); err == nil {
|
||||
t.Fatal("expected to fail")
|
||||
}
|
||||
if _, err = initFormatFS(disk); err == nil {
|
||||
t.Fatal("expected to fail")
|
||||
}
|
||||
|
||||
// With unknown formatMetaV1.Version formatMetaGetFormatBackendFS, initFormatFS should return error.
|
||||
if err = f.Truncate(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Here we set formatMetaV1.Version to "2"
|
||||
if _, err = f.WriteString(`{"version":"2","format":"fs","fs":{"version":"1"}}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = formatMetaGetFormatBackendFS(f); err == nil {
|
||||
t.Fatal("expected to fail")
|
||||
}
|
||||
if _, err = initFormatFS(disk); err == nil {
|
||||
t.Fatal("expected to fail")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
// Format related consts
|
||||
const (
|
||||
// Format config file carries backend format specific details.
|
||||
formatConfigFile = "format.json"
|
||||
|
||||
// Format config tmp file carries backend format.
|
||||
formatConfigFileTmp = "format.json.tmp"
|
||||
)
|
||||
|
||||
const (
|
||||
// Version of the formatMetaV1
|
||||
formatMetaVersionV1 = "1"
|
||||
)
|
||||
|
||||
// format.json currently has the format:
|
||||
// {
|
||||
// "version": "1",
|
||||
// "format": "XXXXX",
|
||||
// "XXXXX": {
|
||||
//
|
||||
// }
|
||||
// }
|
||||
// Here "XXXXX" depends on the backend, currently we have "fs" and "xl" implementations.
|
||||
// formatMetaV1 should be inherited by backend format structs. Please look at format-fs.go
|
||||
// and format-xl.go for details.
|
||||
|
||||
// Ideally we will never have a situation where we will have to change the
|
||||
// fields of this struct and deal with related migration.
|
||||
type formatMetaV1 struct {
|
||||
// Version of the format config.
|
||||
Version string `json:"version"`
|
||||
// Format indicates the backend format type, supports two values 'xl' and 'fs'.
|
||||
Format string `json:"format"`
|
||||
}
|
||||
@@ -20,167 +20,30 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"reflect"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// fsFormat - structure holding 'fs' format.
|
||||
type fsFormat struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
|
||||
// FS format version strings.
|
||||
const (
|
||||
// Represents the current backend disk structure
|
||||
// version under `.minio.sys` and actual data namespace.
|
||||
|
||||
// formatConfigV1.fsFormat.Version
|
||||
fsFormatBackendV1 = "1"
|
||||
)
|
||||
|
||||
// xlFormat - structure holding 'xl' format.
|
||||
type xlFormat struct {
|
||||
Version string `json:"version"` // Version of 'xl' format.
|
||||
Disk string `json:"disk"` // Disk field carries assigned disk uuid.
|
||||
// JBOD field carries the input disk order generated the first
|
||||
// time when fresh disks were supplied.
|
||||
JBOD []string `json:"jbod"`
|
||||
}
|
||||
|
||||
// XL format version strings.
|
||||
const (
|
||||
// Represents the current backend disk structure
|
||||
// version under `.minio.sys` and actual data namespace.
|
||||
|
||||
// formatConfigV1.xlFormat.Version
|
||||
xlFormatBackendV1 = "1"
|
||||
)
|
||||
|
||||
// formatConfigV1 - structure holds format config version '1'.
|
||||
type formatConfigV1 struct {
|
||||
Version string `json:"version"` // Version of the format config.
|
||||
// Format indicates the backend format type, supports two values 'xl' and 'fs'.
|
||||
Format string `json:"format"`
|
||||
FS *fsFormat `json:"fs,omitempty"` // FS field holds fs format.
|
||||
XL *xlFormat `json:"xl,omitempty"` // XL field holds xl format.
|
||||
}
|
||||
|
||||
// Format json file.
|
||||
const (
|
||||
// Format config file carries backend format specific details.
|
||||
formatConfigFile = "format.json"
|
||||
|
||||
// Format config tmp file carries backend format.
|
||||
formatConfigFileTmp = "format.json.tmp"
|
||||
)
|
||||
|
||||
// `format.json` version value.
|
||||
const (
|
||||
// formatConfigV1.Version represents the version string
|
||||
// of the current structure and its fields in `format.json`.
|
||||
formatFileV1 = "1"
|
||||
|
||||
// Future `format.json` structure changes should have
|
||||
// its own version and should be subsequently listed here.
|
||||
)
|
||||
|
||||
// Constitutes `format.json` backend name.
|
||||
const (
|
||||
// Represents FS backend.
|
||||
formatBackendFS = "fs"
|
||||
|
||||
// Represents XL backend.
|
||||
formatBackendXL = "xl"
|
||||
|
||||
// formatXLV1.XL.Version
|
||||
formatXLVersionV1 = "1"
|
||||
)
|
||||
|
||||
// CheckFS if the format is FS and is valid with right values
|
||||
// returns appropriate errors otherwise.
|
||||
func (f *formatConfigV1) CheckFS() error {
|
||||
// Validate if format config version is v1.
|
||||
if f.Version != formatFileV1 {
|
||||
return fmt.Errorf("Unknown format file version '%s'", f.Version)
|
||||
}
|
||||
|
||||
// Validate if we have the expected format.
|
||||
if f.Format != formatBackendFS {
|
||||
return fmt.Errorf("FS backend format required. Found '%s'", f.Format)
|
||||
}
|
||||
|
||||
// Check if format is currently supported.
|
||||
if f.FS.Version != fsFormatBackendV1 {
|
||||
return fmt.Errorf("Unknown backend FS format version '%s'", f.FS.Version)
|
||||
}
|
||||
|
||||
// Success.
|
||||
return nil
|
||||
}
|
||||
|
||||
// LoadFormat - loads format config v1, returns `errUnformattedDisk`
|
||||
// if reading format.json fails with io.EOF.
|
||||
func (f *formatConfigV1) LoadFormat(lk *lock.LockedFile) error {
|
||||
_, err := f.ReadFrom(lk)
|
||||
if errorCause(err) == io.EOF {
|
||||
// No data on disk `format.json` still empty
|
||||
// treat it as unformatted disk.
|
||||
return traceError(errUnformattedDisk)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (f *formatConfigV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
|
||||
// Serialize to prepare to write to disk.
|
||||
var fbytes []byte
|
||||
fbytes, err = json.Marshal(f)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
if err = lk.Truncate(0); err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
_, err = lk.Write(fbytes)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
return int64(len(fbytes)), nil
|
||||
}
|
||||
|
||||
func (f *formatConfigV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
|
||||
var fbytes []byte
|
||||
fi, err := lk.Stat()
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
fbytes, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
if len(fbytes) == 0 {
|
||||
return 0, traceError(io.EOF)
|
||||
}
|
||||
// Decode `format.json`.
|
||||
if err = json.Unmarshal(fbytes, f); err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
return int64(len(fbytes)), nil
|
||||
}
|
||||
|
||||
func newFSFormat() (format *formatConfigV1) {
|
||||
return newFSFormatV1()
|
||||
}
|
||||
|
||||
// newFSFormatV1 - initializes new formatConfigV1 with FS format info.
|
||||
func newFSFormatV1() (format *formatConfigV1) {
|
||||
return &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
}
|
||||
// Represents the current backend disk structure
|
||||
// version under `.minio.sys` and actual data namespace.
|
||||
// formatXLV1 - structure holds format config version '1'.
|
||||
type formatXLV1 struct {
|
||||
formatMetaV1
|
||||
XL struct {
|
||||
Version string `json:"version"` // Version of 'xl' format.
|
||||
Disk string `json:"disk"` // Disk field carries assigned disk uuid.
|
||||
// JBOD field carries the input disk order generated the first
|
||||
// time when fresh disks were supplied.
|
||||
JBOD []string `json:"jbod"`
|
||||
} `json:"xl"` // XL field holds xl format.
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -270,7 +133,7 @@ func formatErrsSummary(errs []error) (formatCount, unformattedDiskCount,
|
||||
}
|
||||
|
||||
// loadAllFormats - load all format config from all input disks in parallel.
|
||||
func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
|
||||
func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatXLV1, []error) {
|
||||
// Initialize sync waitgroup.
|
||||
var wg = &sync.WaitGroup{}
|
||||
|
||||
@@ -278,38 +141,32 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
|
||||
var sErrs = make([]error, len(bootstrapDisks))
|
||||
|
||||
// Initialize format configs.
|
||||
var formatConfigs = make([]*formatConfigV1, len(bootstrapDisks))
|
||||
var formats = make([]*formatXLV1, len(bootstrapDisks))
|
||||
|
||||
// Make a volume entry on all underlying storage disks.
|
||||
// Load format from each disk in parallel
|
||||
for index, disk := range bootstrapDisks {
|
||||
if disk == nil {
|
||||
sErrs[index] = errDiskNotFound
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
// Make a volume inside a go-routine.
|
||||
// Launch go-routine per disk.
|
||||
go func(index int, disk StorageAPI) {
|
||||
defer wg.Done()
|
||||
formatConfig, lErr := loadFormat(disk)
|
||||
format, lErr := loadFormat(disk)
|
||||
if lErr != nil {
|
||||
sErrs[index] = lErr
|
||||
return
|
||||
}
|
||||
formatConfigs[index] = formatConfig
|
||||
formats[index] = format
|
||||
}(index, disk)
|
||||
}
|
||||
|
||||
// Wait for all make vol to finish.
|
||||
// Wait for all go-routines to finish.
|
||||
wg.Wait()
|
||||
|
||||
for _, err := range sErrs {
|
||||
if err != nil {
|
||||
// Return all formats and errors.
|
||||
return formatConfigs, sErrs
|
||||
}
|
||||
}
|
||||
// Return all formats and nil
|
||||
return formatConfigs, sErrs
|
||||
return formats, sErrs
|
||||
}
|
||||
|
||||
// genericFormatCheckXL - validates and returns error.
|
||||
@@ -317,7 +174,7 @@ func loadAllFormats(bootstrapDisks []StorageAPI) ([]*formatConfigV1, []error) {
|
||||
// if (any disk is corrupt) return error // phase2
|
||||
// if (jbod inconsistent) return error // phase2
|
||||
// if (disks not recognized) // Always error.
|
||||
func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err error) {
|
||||
func genericFormatCheckXL(formats []*formatXLV1, sErrs []error) (err error) {
|
||||
// Calculate the errors.
|
||||
var (
|
||||
errCorruptFormatCount = 0
|
||||
@@ -341,20 +198,20 @@ func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err e
|
||||
}
|
||||
|
||||
// Calculate read quorum.
|
||||
readQuorum := len(formatConfigs) / 2
|
||||
readQuorum := len(formats) / 2
|
||||
|
||||
// Validate the err count under read quorum.
|
||||
if errCount > len(formatConfigs)-readQuorum {
|
||||
if errCount > len(formats)-readQuorum {
|
||||
return errXLReadQuorum
|
||||
}
|
||||
|
||||
// Check if number of corrupted format under read quorum
|
||||
if errCorruptFormatCount > len(formatConfigs)-readQuorum {
|
||||
if errCorruptFormatCount > len(formats)-readQuorum {
|
||||
return errCorruptedFormat
|
||||
}
|
||||
|
||||
// Validates if format and JBOD are consistent across all disks.
|
||||
if err = checkFormatXL(formatConfigs); err != nil {
|
||||
if err = checkFormatXL(formats); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -365,15 +222,15 @@ func genericFormatCheckXL(formatConfigs []*formatConfigV1, sErrs []error) (err e
|
||||
// isSavedUUIDInOrder - validates if disk uuid is present and valid in all
|
||||
// available format config JBOD. This function also validates if the disk UUID
|
||||
// is always available on all JBOD under the same order.
|
||||
func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool {
|
||||
func isSavedUUIDInOrder(uuid string, formats []*formatXLV1) bool {
|
||||
var orderIndexes []int
|
||||
// Validate each for format.json for relevant uuid.
|
||||
for _, formatConfig := range formatConfigs {
|
||||
if formatConfig == nil {
|
||||
for _, format := range formats {
|
||||
if format == nil {
|
||||
continue
|
||||
}
|
||||
// Validate if UUID is present in JBOD.
|
||||
uuidIndex := findDiskIndex(uuid, formatConfig.XL.JBOD)
|
||||
uuidIndex := findDiskIndex(uuid, format.XL.JBOD)
|
||||
if uuidIndex == -1 {
|
||||
// UUID not found.
|
||||
errorIf(errDiskNotFound, "Disk %s not found in JBOD list", uuid)
|
||||
@@ -397,15 +254,15 @@ func isSavedUUIDInOrder(uuid string, formatConfigs []*formatConfigV1) bool {
|
||||
}
|
||||
|
||||
// checkDisksConsistency - checks if all disks are consistent with all JBOD entries on all disks.
|
||||
func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
|
||||
var disks = make([]string, len(formatConfigs))
|
||||
func checkDisksConsistency(formats []*formatXLV1) error {
|
||||
var disks = make([]string, len(formats))
|
||||
// Collect currently available disk uuids.
|
||||
for index, formatConfig := range formatConfigs {
|
||||
if formatConfig == nil {
|
||||
for index, format := range formats {
|
||||
if format == nil {
|
||||
disks[index] = ""
|
||||
continue
|
||||
}
|
||||
disks[index] = formatConfig.XL.Disk
|
||||
disks[index] = format.XL.Disk
|
||||
}
|
||||
// Validate collected uuids and verify JBOD.
|
||||
for _, uuid := range disks {
|
||||
@@ -413,7 +270,7 @@ func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
|
||||
continue
|
||||
}
|
||||
// Is uuid present on all JBOD ?.
|
||||
if !isSavedUUIDInOrder(uuid, formatConfigs) {
|
||||
if !isSavedUUIDInOrder(uuid, formats) {
|
||||
return fmt.Errorf("%s disk not found in JBOD", uuid)
|
||||
}
|
||||
}
|
||||
@@ -421,17 +278,17 @@ func checkDisksConsistency(formatConfigs []*formatConfigV1) error {
|
||||
}
|
||||
|
||||
// checkJBODConsistency - validate xl jbod order if they are consistent.
|
||||
func checkJBODConsistency(formatConfigs []*formatConfigV1) error {
|
||||
func checkJBODConsistency(formats []*formatXLV1) error {
|
||||
var sentinelJBOD []string
|
||||
// Extract first valid JBOD.
|
||||
for _, format := range formatConfigs {
|
||||
for _, format := range formats {
|
||||
if format == nil {
|
||||
continue
|
||||
}
|
||||
sentinelJBOD = format.XL.JBOD
|
||||
break
|
||||
}
|
||||
for _, format := range formatConfigs {
|
||||
for _, format := range formats {
|
||||
if format == nil {
|
||||
continue
|
||||
}
|
||||
@@ -457,14 +314,14 @@ func findDiskIndex(disk string, jbod []string) int {
|
||||
// format-config. If assignUUIDs is true, it assigns UUIDs to disks
|
||||
// with missing format configurations in the reference configuration.
|
||||
func reorderDisks(bootstrapDisks []StorageAPI,
|
||||
formatConfigs []*formatConfigV1, assignUUIDs bool) (*formatConfigV1,
|
||||
formats []*formatXLV1, assignUUIDs bool) (*formatXLV1,
|
||||
[]StorageAPI, error) {
|
||||
|
||||
// Pick first non-nil format-cfg as reference
|
||||
var refCfg *formatConfigV1
|
||||
for _, formatConfig := range formatConfigs {
|
||||
if formatConfig != nil {
|
||||
refCfg = formatConfig
|
||||
var refCfg *formatXLV1
|
||||
for _, format := range formats {
|
||||
if format != nil {
|
||||
refCfg = format
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -475,7 +332,7 @@ func reorderDisks(bootstrapDisks []StorageAPI,
|
||||
|
||||
// construct reordered disk slice
|
||||
var newDisks = make([]StorageAPI, len(bootstrapDisks))
|
||||
for fIndex, format := range formatConfigs {
|
||||
for fIndex, format := range formats {
|
||||
if format == nil {
|
||||
continue
|
||||
}
|
||||
@@ -500,7 +357,7 @@ func reorderDisks(bootstrapDisks []StorageAPI,
|
||||
}
|
||||
|
||||
// loadFormat - loads format.json from disk.
|
||||
func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
|
||||
func loadFormat(disk StorageAPI) (format *formatXLV1, err error) {
|
||||
buf, err := disk.ReadAll(minioMetaBucket, formatConfigFile)
|
||||
if err != nil {
|
||||
// 'file not found' and 'volume not found' as
|
||||
@@ -511,8 +368,10 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(vols) > 1 {
|
||||
// 'format.json' not found, but we found user data.
|
||||
if len(vols) > 1 || (len(vols) == 1 &&
|
||||
vols[0].Name != minioMetaBucket) {
|
||||
// 'format.json' not found, but we
|
||||
// found user data.
|
||||
return nil, errCorruptedFormat
|
||||
}
|
||||
// No other data found, its a fresh disk.
|
||||
@@ -522,7 +381,7 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
|
||||
}
|
||||
|
||||
// Try to decode format json into formatConfigV1 struct.
|
||||
format = &formatConfigV1{}
|
||||
format = &formatXLV1{}
|
||||
if err = json.Unmarshal(buf, format); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -531,26 +390,22 @@ func loadFormat(disk StorageAPI) (format *formatConfigV1, err error) {
|
||||
return format, nil
|
||||
}
|
||||
|
||||
// collectNSaveNewFormatConfigs - creates new format configs based on
|
||||
// the reference config and saves it on all disks, this is to be
|
||||
// called from healFormatXL* functions.
|
||||
func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
|
||||
orderedDisks []StorageAPI) error {
|
||||
// collectNSaveNewFormatConfigs - generates new format configs based on
|
||||
// the given ref. config and saves on each disk
|
||||
func collectNSaveNewFormatConfigs(referenceConfig *formatXLV1,
|
||||
orderedDisks []StorageAPI, dryRun bool) error {
|
||||
|
||||
// Collect new format configs that need to be written.
|
||||
var newFormatConfigs = make([]*formatConfigV1, len(orderedDisks))
|
||||
var newFormatConfigs = make([]*formatXLV1, len(orderedDisks))
|
||||
for index := range orderedDisks {
|
||||
// New configs are generated since we are going
|
||||
// to re-populate across all disks.
|
||||
config := &formatConfigV1{
|
||||
Version: referenceConfig.Version,
|
||||
Format: referenceConfig.Format,
|
||||
XL: &xlFormat{
|
||||
Version: referenceConfig.XL.Version,
|
||||
Disk: referenceConfig.XL.JBOD[index],
|
||||
JBOD: referenceConfig.XL.JBOD,
|
||||
},
|
||||
}
|
||||
config := &formatXLV1{}
|
||||
config.Version = referenceConfig.Version
|
||||
config.Format = referenceConfig.Format
|
||||
config.XL.Version = referenceConfig.XL.Version
|
||||
config.XL.Disk = referenceConfig.XL.JBOD[index]
|
||||
config.XL.JBOD = referenceConfig.XL.JBOD
|
||||
newFormatConfigs[index] = config
|
||||
}
|
||||
|
||||
@@ -560,18 +415,21 @@ func collectNSaveNewFormatConfigs(referenceConfig *formatConfigV1,
|
||||
}
|
||||
|
||||
// Save new `format.json` across all disks, in JBOD order.
|
||||
return saveFormatXL(orderedDisks, newFormatConfigs)
|
||||
if !dryRun {
|
||||
return saveFormatXL(orderedDisks, newFormatConfigs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Heals any missing format.json on the drives. Returns error only for
|
||||
// unexpected errors as regular errors can be ignored since there
|
||||
// might be enough quorum to be operational. Heals only fresh disks.
|
||||
func healFormatXLFreshDisks(storageDisks []StorageAPI,
|
||||
formatConfigs []*formatConfigV1) error {
|
||||
formats []*formatXLV1, dryRun bool) error {
|
||||
|
||||
// Reorder the disks based on the JBOD order.
|
||||
// Reorder disks based on JBOD order, and get reference config.
|
||||
referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
|
||||
formatConfigs, true)
|
||||
formats, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -580,22 +438,25 @@ func healFormatXLFreshDisks(storageDisks []StorageAPI,
|
||||
// We need to make sure we have kept the previous order
|
||||
// and allowed fresh disks to be arranged anywhere.
|
||||
// Following block facilitates to put fresh disks.
|
||||
for index, format := range formatConfigs {
|
||||
for index, format := range formats {
|
||||
if format != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
// Format is missing so we go through ordered disks.
|
||||
if format == nil {
|
||||
// At this point when disk is missing the fresh disk
|
||||
// in the stack get it back from storageDisks.
|
||||
for oIndex, disk := range orderedDisks {
|
||||
if disk == nil {
|
||||
orderedDisks[oIndex] = storageDisks[index]
|
||||
break
|
||||
}
|
||||
// At this point when disk is missing the fresh disk
|
||||
// in the stack get it back from storageDisks.
|
||||
for oIndex, disk := range orderedDisks {
|
||||
if disk == nil {
|
||||
orderedDisks[oIndex] = storageDisks[index]
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// apply new format config and save to all disks
|
||||
return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
|
||||
return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks,
|
||||
dryRun)
|
||||
}
|
||||
|
||||
// collectUnAssignedDisks - collect disks unassigned to orderedDisks
|
||||
@@ -623,9 +484,9 @@ func collectUnAssignedDisks(storageDisks, orderedDisks []StorageAPI) (
|
||||
// Inspect the content of all disks to guess the right order according
|
||||
// to the format files. The right order is represented in orderedDisks
|
||||
func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
|
||||
formatConfigs []*formatConfigV1) ([]StorageAPI, error) {
|
||||
formats []*formatXLV1) ([]StorageAPI, error) {
|
||||
|
||||
for index, format := range formatConfigs {
|
||||
for index, format := range formats {
|
||||
if format != nil {
|
||||
continue
|
||||
}
|
||||
@@ -680,11 +541,11 @@ func reorderDisksByInspection(orderedDisks, storageDisks []StorageAPI,
|
||||
|
||||
// Heals corrupted format json in all disks
|
||||
func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
|
||||
formatConfigs []*formatConfigV1) error {
|
||||
formats []*formatXLV1, dryRun bool) error {
|
||||
|
||||
// Reorder the disks based on the JBOD order.
|
||||
// Reorder disks based on JBOD order, and update ref. config.
|
||||
referenceConfig, orderedDisks, err := reorderDisks(storageDisks,
|
||||
formatConfigs, true)
|
||||
formats, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -692,7 +553,7 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
|
||||
// For disks with corrupted formats, inspect the disks
|
||||
// contents to guess the disks order
|
||||
orderedDisks, err = reorderDisksByInspection(orderedDisks, storageDisks,
|
||||
formatConfigs)
|
||||
formats)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -710,8 +571,9 @@ func healFormatXLCorruptedDisks(storageDisks []StorageAPI,
|
||||
}
|
||||
}
|
||||
|
||||
// apply new format config and save to all disks
|
||||
return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks)
|
||||
// generate and write new configs to all disks
|
||||
return collectNSaveNewFormatConfigs(referenceConfig, orderedDisks,
|
||||
dryRun)
|
||||
}
|
||||
|
||||
// loadFormatXL - loads XL `format.json` and returns back properly
|
||||
@@ -720,7 +582,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
|
||||
var unformattedDisksFoundCnt = 0
|
||||
var diskNotFoundCount = 0
|
||||
var corruptedDisksFoundCnt = 0
|
||||
formatConfigs := make([]*formatConfigV1, len(bootstrapDisks))
|
||||
formats := make([]*formatXLV1, len(bootstrapDisks))
|
||||
|
||||
// Try to load `format.json` bootstrap disks.
|
||||
for index, disk := range bootstrapDisks {
|
||||
@@ -728,7 +590,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
|
||||
diskNotFoundCount++
|
||||
continue
|
||||
}
|
||||
var formatXL *formatConfigV1
|
||||
var formatXL *formatXLV1
|
||||
formatXL, err = loadFormat(disk)
|
||||
if err != nil {
|
||||
if err == errUnformattedDisk {
|
||||
@@ -744,7 +606,7 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
|
||||
return nil, err
|
||||
}
|
||||
// Save valid formats.
|
||||
formatConfigs[index] = formatXL
|
||||
formats[index] = formatXL
|
||||
}
|
||||
|
||||
// If all disks indicate that 'format.json' is not available return 'errUnformattedDisk'.
|
||||
@@ -759,59 +621,59 @@ func loadFormatXL(bootstrapDisks []StorageAPI, readQuorum int) (disks []StorageA
|
||||
}
|
||||
|
||||
// Validate the format configs read are correct.
|
||||
if err = checkFormatXL(formatConfigs); err != nil {
|
||||
if err = checkFormatXL(formats); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Erasure code requires disks to be presented in the same
|
||||
// order each time.
|
||||
_, orderedDisks, err := reorderDisks(bootstrapDisks, formatConfigs,
|
||||
_, orderedDisks, err := reorderDisks(bootstrapDisks, formats,
|
||||
false)
|
||||
return orderedDisks, err
|
||||
}
|
||||
|
||||
func checkFormatXLValue(formatXL *formatConfigV1) error {
|
||||
func checkFormatXLValue(formatXL *formatXLV1) error {
|
||||
// Validate format version and format type.
|
||||
if formatXL.Version != formatFileV1 {
|
||||
if formatXL.Version != formatMetaVersionV1 {
|
||||
return fmt.Errorf("Unsupported version of backend format [%s] found", formatXL.Version)
|
||||
}
|
||||
if formatXL.Format != formatBackendXL {
|
||||
return fmt.Errorf("Unsupported backend format [%s] found", formatXL.Format)
|
||||
}
|
||||
if formatXL.XL.Version != "1" {
|
||||
if formatXL.XL.Version != formatXLVersionV1 {
|
||||
return fmt.Errorf("Unsupported XL backend format found [%s]", formatXL.XL.Version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFormatXLValues(formatConfigs []*formatConfigV1) (int, error) {
|
||||
for i, formatXL := range formatConfigs {
|
||||
func checkFormatXLValues(formats []*formatXLV1) (int, error) {
|
||||
for i, formatXL := range formats {
|
||||
if formatXL == nil {
|
||||
continue
|
||||
}
|
||||
if err := checkFormatXLValue(formatXL); err != nil {
|
||||
return i, err
|
||||
}
|
||||
if len(formatConfigs) != len(formatXL.XL.JBOD) {
|
||||
if len(formats) != len(formatXL.XL.JBOD) {
|
||||
return i, fmt.Errorf("Number of disks %d did not match the backend format %d",
|
||||
len(formatConfigs), len(formatXL.XL.JBOD))
|
||||
len(formats), len(formatXL.XL.JBOD))
|
||||
}
|
||||
}
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
// checkFormatXL - verifies if format.json format is intact.
|
||||
func checkFormatXL(formatConfigs []*formatConfigV1) error {
|
||||
if _, err := checkFormatXLValues(formatConfigs); err != nil {
|
||||
func checkFormatXL(formats []*formatXLV1) error {
|
||||
if _, err := checkFormatXLValues(formats); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := checkJBODConsistency(formatConfigs); err != nil {
|
||||
if err := checkJBODConsistency(formats); err != nil {
|
||||
return err
|
||||
}
|
||||
return checkDisksConsistency(formatConfigs)
|
||||
return checkDisksConsistency(formats)
|
||||
}
|
||||
|
||||
// saveFormatXL - populates `format.json` on disks in its order.
|
||||
func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
|
||||
func saveFormatXL(storageDisks []StorageAPI, formats []*formatXLV1) error {
|
||||
var errs = make([]error, len(storageDisks))
|
||||
var wg = &sync.WaitGroup{}
|
||||
// Write `format.json` to all disks.
|
||||
@@ -820,7 +682,7 @@ func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(index int, disk StorageAPI, format *formatConfigV1) {
|
||||
go func(index int, disk StorageAPI, format *formatXLV1) {
|
||||
defer wg.Done()
|
||||
|
||||
// Marshal and write to disk.
|
||||
@@ -861,45 +723,37 @@ func saveFormatXL(storageDisks []StorageAPI, formats []*formatConfigV1) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// initFormatXL - save XL format configuration on all disks.
|
||||
func initFormatXL(storageDisks []StorageAPI) (err error) {
|
||||
// Initialize jbods.
|
||||
var jbod = make([]string, len(storageDisks))
|
||||
// Return a slice of format, to be used to format uninitialized disks.
|
||||
func newFormatXLV1(diskCount int) []*formatXLV1 {
|
||||
var jbod = make([]string, diskCount)
|
||||
|
||||
// Initialize formats.
|
||||
var formats = make([]*formatConfigV1, len(storageDisks))
|
||||
var formats = make([]*formatXLV1, diskCount)
|
||||
|
||||
// Initialize `format.json`.
|
||||
for index, disk := range storageDisks {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
// Allocate format config.
|
||||
formats[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: mustGetUUID(),
|
||||
},
|
||||
}
|
||||
jbod[index] = formats[index].XL.Disk
|
||||
for i := 0; i < diskCount; i++ {
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = mustGetUUID()
|
||||
formats[i] = format
|
||||
jbod[i] = formats[i].XL.Disk
|
||||
}
|
||||
|
||||
// Update the jbod entries.
|
||||
for index, disk := range storageDisks {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
// Save jbod.
|
||||
formats[index].XL.JBOD = jbod
|
||||
for i := 0; i < diskCount; i++ {
|
||||
formats[i].XL.JBOD = jbod
|
||||
}
|
||||
|
||||
return formats
|
||||
}
|
||||
|
||||
// initFormatXL - save XL format configuration on all disks.
|
||||
func initFormatXL(storageDisks []StorageAPI) (err error) {
|
||||
// Initialize meta volume, if volume already exists ignores it.
|
||||
if err := initMetaVolume(storageDisks); err != nil {
|
||||
return fmt.Errorf("Unable to initialize '.minio.sys' meta volume, %s", err)
|
||||
}
|
||||
|
||||
// Save formats `format.json` across all disks.
|
||||
return saveFormatXL(storageDisks, formats)
|
||||
return saveFormatXL(storageDisks, newFormatXLV1(len(storageDisks)))
|
||||
}
|
||||
@@ -18,53 +18,46 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// generates a valid format.json for XL backend.
|
||||
func genFormatXLValid() []*formatConfigV1 {
|
||||
func genFormatXLValid() []*formatXLV1 {
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for XL backend.
|
||||
func genFormatXLInvalidVersion() []*formatConfigV1 {
|
||||
func genFormatXLInvalidVersion() []*formatXLV1 {
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Corrupt version numbers.
|
||||
formatConfigs[0].Version = "2"
|
||||
@@ -73,46 +66,42 @@ func genFormatXLInvalidVersion() []*formatConfigV1 {
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for XL backend.
|
||||
func genFormatXLInvalidFormat() []*formatConfigV1 {
|
||||
func genFormatXLInvalidFormat() []*formatXLV1 {
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Corrupt version numbers.
|
||||
// Corrupt format.
|
||||
formatConfigs[0].Format = "lx"
|
||||
formatConfigs[3].Format = "lx"
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for XL backend.
|
||||
func genFormatXLInvalidXLVersion() []*formatConfigV1 {
|
||||
func genFormatXLInvalidXLVersion() []*formatXLV1 {
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Corrupt version numbers.
|
||||
formatConfigs[0].XL.Version = "10"
|
||||
@@ -120,51 +109,40 @@ func genFormatXLInvalidXLVersion() []*formatConfigV1 {
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
func genFormatFS() *formatConfigV1 {
|
||||
return &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
}
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for XL backend.
|
||||
func genFormatXLInvalidJBODCount() []*formatConfigV1 {
|
||||
func genFormatXLInvalidJBODCount() []*formatXLV1 {
|
||||
jbod := make([]string, 7)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json JBOD for XL backend.
|
||||
func genFormatXLInvalidJBOD() []*formatConfigV1 {
|
||||
func genFormatXLInvalidJBOD() []*formatXLV1 {
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
@@ -176,22 +154,20 @@ func genFormatXLInvalidJBOD() []*formatConfigV1 {
|
||||
}
|
||||
|
||||
// generates a invalid format.json Disk UUID for XL backend.
|
||||
func genFormatXLInvalidDisks() []*formatConfigV1 {
|
||||
func genFormatXLInvalidDisks() []*formatXLV1 {
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Make disk 5 and disk 8 have inconsistent disk uuid's.
|
||||
formatConfigs[4].XL.Disk = mustGetUUID()
|
||||
@@ -200,22 +176,20 @@ func genFormatXLInvalidDisks() []*formatConfigV1 {
|
||||
}
|
||||
|
||||
// generates a invalid format.json Disk UUID in wrong order for XL backend.
|
||||
func genFormatXLInvalidDisksOrder() []*formatConfigV1 {
|
||||
func genFormatXLInvalidDisksOrder() []*formatXLV1 {
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Re order jbod for failure case.
|
||||
var jbod1 = make([]string, 8)
|
||||
@@ -297,7 +271,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
|
||||
formatConfigs, _ := loadAllFormats(storageDisks)
|
||||
|
||||
// Start healing disks
|
||||
err = healFormatXLFreshDisks(storageDisks, formatConfigs)
|
||||
err = healFormatXLFreshDisks(storageDisks, formatConfigs, false)
|
||||
if err != nil {
|
||||
t.Fatal("healing corrupted disk failed: ", err)
|
||||
}
|
||||
@@ -316,7 +290,7 @@ func TestFormatXLHealFreshDisks(t *testing.T) {
|
||||
// a given disk to test healing a corrupted disk
|
||||
func TestFormatXLHealCorruptedDisks(t *testing.T) {
|
||||
// Create an instance of xl backend.
|
||||
obj, fsDirs, err := prepareXL()
|
||||
obj, fsDirs, err := prepareXL16()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -371,7 +345,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
|
||||
formatConfigs, _ := loadAllFormats(permutedStorageDisks)
|
||||
|
||||
// Start healing disks
|
||||
err = healFormatXLCorruptedDisks(permutedStorageDisks, formatConfigs)
|
||||
err = healFormatXLCorruptedDisks(permutedStorageDisks, formatConfigs, false)
|
||||
if err != nil {
|
||||
t.Fatal("healing corrupted disk failed: ", err)
|
||||
}
|
||||
@@ -390,7 +364,7 @@ func TestFormatXLHealCorruptedDisks(t *testing.T) {
|
||||
// some of format.json
|
||||
func TestFormatXLReorderByInspection(t *testing.T) {
|
||||
// Create an instance of xl backend.
|
||||
obj, fsDirs, err := prepareXL()
|
||||
obj, fsDirs, err := prepareXL16()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -458,7 +432,7 @@ func TestFormatXLReorderByInspection(t *testing.T) {
|
||||
// - invalid JBOD
|
||||
// - invalid Disk uuid
|
||||
func TestFormatXL(t *testing.T) {
|
||||
formatInputCases := [][]*formatConfigV1{
|
||||
formatInputCases := [][]*formatXLV1{
|
||||
genFormatXLValid(),
|
||||
genFormatXLInvalidVersion(),
|
||||
genFormatXLInvalidFormat(),
|
||||
@@ -469,7 +443,7 @@ func TestFormatXL(t *testing.T) {
|
||||
genFormatXLInvalidDisksOrder(),
|
||||
}
|
||||
testCases := []struct {
|
||||
formatConfigs []*formatConfigV1
|
||||
formatConfigs []*formatXLV1
|
||||
shouldPass bool
|
||||
}{
|
||||
{
|
||||
@@ -524,22 +498,20 @@ func TestSavedUUIDOrder(t *testing.T) {
|
||||
shouldPass bool
|
||||
}, 8)
|
||||
jbod := make([]string, 8)
|
||||
formatConfigs := make([]*formatConfigV1, 8)
|
||||
formatConfigs := make([]*formatXLV1, 8)
|
||||
for index := range jbod {
|
||||
jbod[index] = mustGetUUID()
|
||||
uuidTestCases[index].uuid = jbod[index]
|
||||
uuidTestCases[index].shouldPass = true
|
||||
}
|
||||
for index := range jbod {
|
||||
formatConfigs[index] = &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendXL,
|
||||
XL: &xlFormat{
|
||||
Version: xlFormatBackendV1,
|
||||
Disk: jbod[index],
|
||||
JBOD: jbod,
|
||||
},
|
||||
}
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendXL
|
||||
format.XL.Version = formatXLVersionV1
|
||||
format.XL.Disk = jbod[index]
|
||||
format.XL.JBOD = jbod
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Re order jbod for failure case.
|
||||
var jbod1 = make([]string, 8)
|
||||
@@ -652,172 +624,18 @@ func TestGenericFormatCheckXL(t *testing.T) {
|
||||
t.Fatalf("Should fail here")
|
||||
}
|
||||
errs = []error{nil}
|
||||
if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil {
|
||||
format := &formatXLV1{}
|
||||
format.Version = formatMetaVersionV1
|
||||
format.Format = formatBackendFS
|
||||
if err := genericFormatCheckXL([]*formatXLV1{format}, errs); err == nil {
|
||||
t.Fatalf("Should fail here")
|
||||
}
|
||||
errs = []error{errFaultyDisk}
|
||||
if err := genericFormatCheckXL([]*formatConfigV1{genFormatFS()}, errs); err == nil {
|
||||
if err := genericFormatCheckXL([]*formatXLV1{format}, errs); err == nil {
|
||||
t.Fatalf("Should fail here")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSCheckFormatFSErr - test loadFormatFS loading older format.
|
||||
func TestFSCheckFormatFSErr(t *testing.T) {
|
||||
// Prepare for testing
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer os.RemoveAll(disk)
|
||||
|
||||
// Assign a new UUID.
|
||||
uuid := mustGetUUID()
|
||||
|
||||
// Initialize meta volume, if volume already exists ignores it.
|
||||
if err := initMetaVolumeFS(disk, uuid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
format *formatConfigV1
|
||||
formatWriteErr error
|
||||
formatCheckErr error
|
||||
shouldPass bool
|
||||
}{
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
},
|
||||
formatCheckErr: nil,
|
||||
shouldPass: true,
|
||||
},
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: "10",
|
||||
},
|
||||
},
|
||||
formatCheckErr: errors.New("Unknown backend FS format version '10'"),
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: formatFileV1,
|
||||
Format: "garbage",
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
},
|
||||
formatCheckErr: errors.New("FS backend format required. Found 'garbage'"),
|
||||
},
|
||||
{
|
||||
format: &formatConfigV1{
|
||||
Version: "-1",
|
||||
Format: formatBackendFS,
|
||||
FS: &fsFormat{
|
||||
Version: fsFormatBackendV1,
|
||||
},
|
||||
},
|
||||
formatCheckErr: errors.New("Unknown format file version '-1'"),
|
||||
},
|
||||
}
|
||||
|
||||
fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
|
||||
for i, testCase := range testCases {
|
||||
lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, err = testCase.format.WriteTo(lk)
|
||||
lk.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Expected nil, got %s", i+1, err)
|
||||
}
|
||||
|
||||
lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
formatCfg := &formatConfigV1{}
|
||||
_, err = formatCfg.ReadFrom(lk)
|
||||
lk.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = formatCfg.CheckFS()
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Should not fail with unexpected %s, expected nil", i+1, err)
|
||||
}
|
||||
if err == nil && !testCase.shouldPass {
|
||||
t.Errorf("Test %d: Should fail with expected %s, got nil", i+1, testCase.formatCheckErr)
|
||||
}
|
||||
if err != nil && !testCase.shouldPass {
|
||||
if errorCause(err).Error() != testCase.formatCheckErr.Error() {
|
||||
t.Errorf("Test %d: Should fail with expected %s, got %s", i+1, testCase.formatCheckErr, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSCheckFormatFS - test loadFormatFS with healty and faulty disks
|
||||
func TestFSCheckFormatFS(t *testing.T) {
|
||||
// Prepare for testing
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer os.RemoveAll(disk)
|
||||
|
||||
// Assign a new UUID.
|
||||
uuid := mustGetUUID()
|
||||
|
||||
// Initialize meta volume, if volume already exists ignores it.
|
||||
if err := initMetaVolumeFS(disk, uuid); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fsFormatPath := pathJoin(disk, minioMetaBucket, formatConfigFile)
|
||||
lk, err := lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
format := newFSFormatV1()
|
||||
_, err = format.WriteTo(lk)
|
||||
lk.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Loading corrupted format file
|
||||
file, err := os.OpenFile((fsFormatPath), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
t.Fatal("Should not fail here", err)
|
||||
}
|
||||
file.Write([]byte{'b'})
|
||||
file.Close()
|
||||
|
||||
lk, err = lock.LockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
format = &formatConfigV1{}
|
||||
_, err = format.ReadFrom(lk)
|
||||
lk.Close()
|
||||
if err == nil {
|
||||
t.Fatal("Should return an error here")
|
||||
}
|
||||
|
||||
// Loading format file from disk not found.
|
||||
os.RemoveAll(disk)
|
||||
_, err = lock.LockedOpenFile((fsFormatPath), os.O_RDONLY, 0600)
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
t.Fatal("Should return 'format.json' does not exist, but got", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadFormatXLErrs(t *testing.T) {
|
||||
nDisks := 16
|
||||
fsDirs, err := getRandomDisks(nDisks)
|
||||
@@ -936,7 +754,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
|
||||
xl := obj.(*xlObjects)
|
||||
formatConfigs, _ := loadAllFormats(xl.storageDisks)
|
||||
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != nil {
|
||||
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs, false); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
|
||||
@@ -959,7 +777,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
|
||||
formatConfigs, _ = loadAllFormats(xl.storageDisks)
|
||||
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
|
||||
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs, false); err != errFaultyDisk {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
@@ -981,7 +799,7 @@ func TestHealFormatXLCorruptedDisksErrs(t *testing.T) {
|
||||
}
|
||||
}
|
||||
formatConfigs, _ = loadAllFormats(xl.storageDisks)
|
||||
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs); err == nil {
|
||||
if err = healFormatXLCorruptedDisks(xl.storageDisks, formatConfigs, false); err == nil {
|
||||
t.Fatal("Should get a json parsing error, ")
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
@@ -1008,7 +826,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
xl := obj.(*xlObjects)
|
||||
formatConfigs, _ := loadAllFormats(xl.storageDisks)
|
||||
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
|
||||
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs, false); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
@@ -1030,7 +848,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
}
|
||||
xl.storageDisks[0] = newNaughtyDisk(posixDisk, nil, errFaultyDisk)
|
||||
formatConfigs, _ = loadAllFormats(xl.storageDisks)
|
||||
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != errFaultyDisk {
|
||||
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs, false); err != errFaultyDisk {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
@@ -1048,7 +866,7 @@ func TestHealFormatXLFreshDisksErrs(t *testing.T) {
|
||||
xl = obj.(*xlObjects)
|
||||
xl.storageDisks[0] = nil
|
||||
formatConfigs, _ = loadAllFormats(xl.storageDisks)
|
||||
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs); err != nil {
|
||||
if err = healFormatXLFreshDisks(xl.storageDisks, formatConfigs, false); err != nil {
|
||||
t.Fatal("Got an unexpected error: ", err)
|
||||
}
|
||||
removeRoots(fsDirs)
|
||||
@@ -1,240 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Error sent by appendParts go-routine when there are holes in parts.
|
||||
// For ex. let's say client uploads part-2 before part-1 in which case we
|
||||
// can not append and have to wait till part-1 is uploaded. Hence we return
|
||||
// this error. Currently this error is not used in the caller.
|
||||
var errPartsMissing = errors.New("required parts missing")
|
||||
|
||||
// Error sent when appendParts go-routine has waited long enough and timedout.
|
||||
var errAppendPartsTimeout = errors.New("appendParts go-routine timeout")
|
||||
|
||||
// Timeout value for the appendParts go-routine.
|
||||
var appendPartsTimeout = 24 * 60 * 60 * time.Second // 24 Hours.
|
||||
|
||||
// Holds a map of uploadID->appendParts go-routine
|
||||
type backgroundAppend struct {
|
||||
sync.Mutex
|
||||
infoMap map[string]bgAppendPartsInfo
|
||||
appendFile io.WriteCloser
|
||||
}
|
||||
|
||||
// Input to the appendParts go-routine
|
||||
type bgAppendPartsInput struct {
|
||||
meta fsMetaV1 // list of parts that need to be appended
|
||||
errCh chan error // error sent by appendParts go-routine
|
||||
}
|
||||
|
||||
// Identifies an appendParts go-routine.
|
||||
type bgAppendPartsInfo struct {
|
||||
inputCh chan bgAppendPartsInput
|
||||
timeoutCh chan struct{} // closed by appendParts go-routine when it timesout
|
||||
abortCh chan struct{} // closed after abort of upload to end the appendParts go-routine
|
||||
completeCh chan struct{} // closed after complete of upload to end the appendParts go-routine
|
||||
}
|
||||
|
||||
// Called after a part is uploaded so that it can be appended in the background.
|
||||
func (fs fsObjects) append(bucket, object, uploadID string, meta fsMetaV1) chan error {
|
||||
fs.bgAppend.Lock()
|
||||
info, ok := fs.bgAppend.infoMap[uploadID]
|
||||
if !ok {
|
||||
// Corresponding appendParts go-routine was not found, create a new one. Would happen when the first
|
||||
// part of a multipart upload is uploaded.
|
||||
inputCh := make(chan bgAppendPartsInput)
|
||||
timeoutCh := make(chan struct{})
|
||||
abortCh := make(chan struct{})
|
||||
completeCh := make(chan struct{})
|
||||
|
||||
info = bgAppendPartsInfo{inputCh, timeoutCh, abortCh, completeCh}
|
||||
fs.bgAppend.infoMap[uploadID] = info
|
||||
|
||||
go fs.appendParts(bucket, object, uploadID, info)
|
||||
}
|
||||
fs.bgAppend.Unlock()
|
||||
|
||||
errCh := make(chan error)
|
||||
go func() {
|
||||
// send input in a goroutine as send on the inputCh can block if appendParts go-routine
|
||||
// is busy appending a part.
|
||||
select {
|
||||
case <-info.timeoutCh:
|
||||
// This is to handle a rare race condition where we found info in b.infoMap
|
||||
// but soon after that appendParts go-routine timed out.
|
||||
errCh <- errAppendPartsTimeout
|
||||
case info.inputCh <- bgAppendPartsInput{meta, errCh}:
|
||||
}
|
||||
}()
|
||||
|
||||
return errCh
|
||||
}
|
||||
|
||||
// Called on complete-multipart-upload. Returns nil if the required parts have been appended.
|
||||
func (fs *fsObjects) complete(bucket, object, uploadID string, meta fsMetaV1) error {
|
||||
fs.bgAppend.Lock()
|
||||
defer fs.bgAppend.Unlock()
|
||||
|
||||
info, ok := fs.bgAppend.infoMap[uploadID]
|
||||
delete(fs.bgAppend.infoMap, uploadID)
|
||||
if !ok {
|
||||
return errPartsMissing
|
||||
}
|
||||
|
||||
errCh := make(chan error)
|
||||
|
||||
select {
|
||||
case <-info.timeoutCh:
|
||||
// This is to handle a rare race condition where we found info in b.infoMap
|
||||
// but soon after that appendParts go-routine timedouted out.
|
||||
return errAppendPartsTimeout
|
||||
case info.inputCh <- bgAppendPartsInput{meta, errCh}:
|
||||
}
|
||||
|
||||
err := <-errCh
|
||||
|
||||
close(info.completeCh)
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
// Called after complete-multipart-upload or abort-multipart-upload so that the appendParts go-routine is not left dangling.
|
||||
func (fs fsObjects) abort(uploadID string) {
|
||||
fs.bgAppend.Lock()
|
||||
defer fs.bgAppend.Unlock()
|
||||
|
||||
info, ok := fs.bgAppend.infoMap[uploadID]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
delete(fs.bgAppend.infoMap, uploadID)
|
||||
|
||||
info.abortCh <- struct{}{}
|
||||
}
|
||||
|
||||
// This is run as a go-routine that appends the parts in the background.
|
||||
func (fs fsObjects) appendParts(bucket, object, uploadID string, info bgAppendPartsInfo) {
|
||||
appendPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID)
|
||||
// Holds the list of parts that is already appended to the "append" file.
|
||||
appendMeta := fsMetaV1{}
|
||||
|
||||
// Allocate staging read buffer.
|
||||
buf := make([]byte, readSizeV1)
|
||||
for {
|
||||
select {
|
||||
case input := <-info.inputCh:
|
||||
// We receive on this channel when new part gets uploaded or when complete-multipart sends
|
||||
// a value on this channel to confirm if all the required parts are appended.
|
||||
meta := input.meta
|
||||
|
||||
for {
|
||||
// Append should be done such a way that if part-3 and part-2 is uploaded before part-1, we
|
||||
// wait till part-1 is uploaded after which we append part-2 and part-3 as well in this for-loop.
|
||||
part, appendNeeded := partToAppend(meta, appendMeta)
|
||||
if !appendNeeded {
|
||||
if reflect.DeepEqual(meta.Parts, appendMeta.Parts) {
|
||||
// Sending nil is useful so that the complete-multipart-upload knows that
|
||||
// all the required parts have been appended.
|
||||
input.errCh <- nil
|
||||
} else {
|
||||
// Sending error is useful so that complete-multipart-upload can fall-back to
|
||||
// its own append process.
|
||||
input.errCh <- errPartsMissing
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
if err := fs.appendPart(bucket, object, uploadID, part, buf); err != nil {
|
||||
fsRemoveFile(appendPath)
|
||||
appendMeta.Parts = nil
|
||||
input.errCh <- err
|
||||
break
|
||||
}
|
||||
|
||||
appendMeta.AddObjectPart(part.Number, part.Name, part.ETag, part.Size)
|
||||
}
|
||||
case <-info.abortCh:
|
||||
// abort-multipart-upload closed abortCh to end the appendParts go-routine.
|
||||
fsRemoveFile(appendPath)
|
||||
|
||||
// So that any racing PutObjectPart does not leave a dangling go-routine.
|
||||
close(info.timeoutCh)
|
||||
|
||||
return
|
||||
case <-info.completeCh:
|
||||
// complete-multipart-upload closed completeCh to end the appendParts go-routine.
|
||||
close(info.timeoutCh) // So that any racing PutObjectPart does not leave a dangling go-routine.
|
||||
return
|
||||
case <-time.After(appendPartsTimeout):
|
||||
// Timeout the goroutine to garbage collect its resources. This would happen if the client initiates
|
||||
// a multipart upload and does not complete/abort it.
|
||||
fs.bgAppend.Lock()
|
||||
delete(fs.bgAppend.infoMap, uploadID)
|
||||
fs.bgAppend.Unlock()
|
||||
|
||||
// Delete the temporary append file as well.
|
||||
fsRemoveFile(appendPath)
|
||||
|
||||
close(info.timeoutCh)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Appends the "part" to the append-file inside "tmp/" that finally gets moved to the actual location
|
||||
// upon complete-multipart-upload.
|
||||
func (fs fsObjects) appendPart(bucket, object, uploadID string, part objectPartInfo, buf []byte) error {
|
||||
partPath := pathJoin(fs.fsPath, minioMetaMultipartBucket, bucket, object, uploadID, part.Name)
|
||||
|
||||
var offset int64
|
||||
// Read each file part to start writing to the temporary concatenated object.
|
||||
file, size, err := fsOpenFile(partPath, offset)
|
||||
if err != nil {
|
||||
if err == errFileNotFound {
|
||||
return errPartsMissing
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
tmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID)
|
||||
// No need to hold a lock, this is a unique file and will be only written
|
||||
// to one one process per uploadID per minio process.
|
||||
wfile, err := os.OpenFile((tmpObjPath), os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer wfile.Close()
|
||||
|
||||
// Fallocate more space as we concatenate.
|
||||
if err = fsFAllocate(int(wfile.Fd()), 0, size); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
_, err = io.CopyBuffer(wfile, file, buf)
|
||||
return err
|
||||
}
|
||||
+63
-111
@@ -1,5 +1,5 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2016 Minio, Inc.
|
||||
* Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
@@ -17,11 +17,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
pathutil "path"
|
||||
"runtime"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// Removes only the file at given path does not remove
|
||||
@@ -29,11 +31,11 @@ import (
|
||||
// windows automatically.
|
||||
func fsRemoveFile(filePath string) (err error) {
|
||||
if filePath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
return errors.Trace(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err = checkPathLength(filePath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
if err = os.Remove((filePath)); err != nil {
|
||||
@@ -47,20 +49,20 @@ func fsRemoveFile(filePath string) (err error) {
|
||||
// long paths for windows automatically.
|
||||
func fsRemoveAll(dirPath string) (err error) {
|
||||
if dirPath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
return errors.Trace(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err = checkPathLength(dirPath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
if err = os.RemoveAll(dirPath); err != nil {
|
||||
if os.IsPermission(err) {
|
||||
return traceError(errVolumeAccessDenied)
|
||||
return errors.Trace(errVolumeAccessDenied)
|
||||
} else if isSysErrNotEmpty(err) {
|
||||
return traceError(errVolumeNotEmpty)
|
||||
return errors.Trace(errVolumeNotEmpty)
|
||||
}
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -70,38 +72,20 @@ func fsRemoveAll(dirPath string) (err error) {
|
||||
// paths for windows automatically.
|
||||
func fsRemoveDir(dirPath string) (err error) {
|
||||
if dirPath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
return errors.Trace(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err = checkPathLength(dirPath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
if err = os.Remove((dirPath)); err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return traceError(errVolumeNotFound)
|
||||
return errors.Trace(errVolumeNotFound)
|
||||
} else if isSysErrNotEmpty(err) {
|
||||
return traceError(errVolumeNotEmpty)
|
||||
return errors.Trace(errVolumeNotEmpty)
|
||||
}
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Creates a new directory, parent dir is also recursively created
|
||||
// if it doesn't exist.
|
||||
func fsMkdirAll(dirPath string) (err error) {
|
||||
if dirPath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err = checkPathLength(dirPath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
if err = os.MkdirAll(dirPath, 0777); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -113,27 +97,27 @@ func fsMkdirAll(dirPath string) (err error) {
|
||||
// are handled automatically.
|
||||
func fsMkdir(dirPath string) (err error) {
|
||||
if dirPath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
return errors.Trace(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err = checkPathLength(dirPath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
if err = os.Mkdir((dirPath), 0777); err != nil {
|
||||
if os.IsExist(err) {
|
||||
return traceError(errVolumeExists)
|
||||
return errors.Trace(errVolumeExists)
|
||||
} else if os.IsPermission(err) {
|
||||
return traceError(errDiskAccessDenied)
|
||||
return errors.Trace(errDiskAccessDenied)
|
||||
} else if isSysErrNotDir(err) {
|
||||
// File path cannot be verified since
|
||||
// one of the parents is a file.
|
||||
return traceError(errDiskAccessDenied)
|
||||
return errors.Trace(errDiskAccessDenied)
|
||||
} else if isSysErrPathNotFound(err) {
|
||||
// Add specific case for windows.
|
||||
return traceError(errDiskAccessDenied)
|
||||
return errors.Trace(errDiskAccessDenied)
|
||||
}
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -146,14 +130,14 @@ func fsMkdir(dirPath string) (err error) {
|
||||
// fsStatFileDir, fsStatFile, fsStatDir.
|
||||
func fsStat(statLoc string) (os.FileInfo, error) {
|
||||
if statLoc == "" {
|
||||
return nil, traceError(errInvalidArgument)
|
||||
return nil, errors.Trace(errInvalidArgument)
|
||||
}
|
||||
if err := checkPathLength(statLoc); err != nil {
|
||||
return nil, traceError(err)
|
||||
return nil, errors.Trace(err)
|
||||
}
|
||||
fi, err := os.Stat((statLoc))
|
||||
if err != nil {
|
||||
return nil, traceError(err)
|
||||
return nil, errors.Trace(err)
|
||||
}
|
||||
|
||||
return fi, nil
|
||||
@@ -163,17 +147,17 @@ func fsStat(statLoc string) (os.FileInfo, error) {
|
||||
func fsStatVolume(volume string) (os.FileInfo, error) {
|
||||
fi, err := fsStat(volume)
|
||||
if err != nil {
|
||||
err = errorCause(err)
|
||||
err = errors.Cause(err)
|
||||
if os.IsNotExist(err) {
|
||||
return nil, traceError(errVolumeNotFound)
|
||||
return nil, errors.Trace(errVolumeNotFound)
|
||||
} else if os.IsPermission(err) {
|
||||
return nil, traceError(errVolumeAccessDenied)
|
||||
return nil, errors.Trace(errVolumeAccessDenied)
|
||||
}
|
||||
return nil, traceError(err)
|
||||
return nil, errors.Trace(err)
|
||||
}
|
||||
|
||||
if !fi.IsDir() {
|
||||
return nil, traceError(errVolumeAccessDenied)
|
||||
return nil, errors.Trace(errVolumeAccessDenied)
|
||||
}
|
||||
|
||||
return fi, nil
|
||||
@@ -187,18 +171,18 @@ func osErrToFSFileErr(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
err = errorCause(err)
|
||||
err = errors.Cause(err)
|
||||
if os.IsNotExist(err) {
|
||||
return traceError(errFileNotFound)
|
||||
return errors.Trace(errFileNotFound)
|
||||
}
|
||||
if os.IsPermission(err) {
|
||||
return traceError(errFileAccessDenied)
|
||||
return errors.Trace(errFileAccessDenied)
|
||||
}
|
||||
if isSysErrNotDir(err) {
|
||||
return traceError(errFileAccessDenied)
|
||||
return errors.Trace(errFileAccessDenied)
|
||||
}
|
||||
if isSysErrPathNotFound(err) {
|
||||
return traceError(errFileNotFound)
|
||||
return errors.Trace(errFileNotFound)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -210,7 +194,7 @@ func fsStatDir(statDir string) (os.FileInfo, error) {
|
||||
return nil, osErrToFSFileErr(err)
|
||||
}
|
||||
if !fi.IsDir() {
|
||||
return nil, traceError(errFileAccessDenied)
|
||||
return nil, errors.Trace(errFileAccessDenied)
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
@@ -222,7 +206,7 @@ func fsStatFile(statFile string) (os.FileInfo, error) {
|
||||
return nil, osErrToFSFileErr(err)
|
||||
}
|
||||
if fi.IsDir() {
|
||||
return nil, traceError(errFileAccessDenied)
|
||||
return nil, errors.Trace(errFileAccessDenied)
|
||||
}
|
||||
return fi, nil
|
||||
}
|
||||
@@ -231,10 +215,10 @@ func fsStatFile(statFile string) (os.FileInfo, error) {
|
||||
// a readable stream and the size of the readable stream.
|
||||
func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
|
||||
if readPath == "" || offset < 0 {
|
||||
return nil, 0, traceError(errInvalidArgument)
|
||||
return nil, 0, errors.Trace(errInvalidArgument)
|
||||
}
|
||||
if err := checkPathLength(readPath); err != nil {
|
||||
return nil, 0, traceError(err)
|
||||
return nil, 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
fr, err := os.Open((readPath))
|
||||
@@ -245,19 +229,19 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
|
||||
// Stat to get the size of the file at path.
|
||||
st, err := os.Stat((readPath))
|
||||
if err != nil {
|
||||
return nil, 0, traceError(err)
|
||||
return nil, 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
// Verify if its not a regular file, since subsequent Seek is undefined.
|
||||
if !st.Mode().IsRegular() {
|
||||
return nil, 0, traceError(errIsNotRegular)
|
||||
return nil, 0, errors.Trace(errIsNotRegular)
|
||||
}
|
||||
|
||||
// Seek to the requested offset.
|
||||
if offset > 0 {
|
||||
_, err = fr.Seek(offset, os.SEEK_SET)
|
||||
if err != nil {
|
||||
return nil, 0, traceError(err)
|
||||
return nil, 0, errors.Trace(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,22 +252,22 @@ func fsOpenFile(readPath string, offset int64) (io.ReadCloser, int64, error) {
|
||||
// Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer.
|
||||
func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) {
|
||||
if filePath == "" || reader == nil {
|
||||
return 0, traceError(errInvalidArgument)
|
||||
return 0, errors.Trace(errInvalidArgument)
|
||||
}
|
||||
|
||||
if err := checkPathLength(filePath); err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(pathutil.Dir(filePath), 0777); err != nil {
|
||||
return 0, traceError(err)
|
||||
if err := mkdirAll(pathutil.Dir(filePath), 0777); err != nil {
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
if err := checkDiskFree(pathutil.Dir(filePath), fallocSize); err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
writer, err := os.OpenFile((filePath), os.O_CREATE|os.O_WRONLY, 0666)
|
||||
writer, err := lock.Open(filePath, os.O_CREATE|os.O_WRONLY, 0666)
|
||||
if err != nil {
|
||||
return 0, osErrToFSFileErr(err)
|
||||
}
|
||||
@@ -292,7 +276,7 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
|
||||
// Fallocate only if the size is final object is known.
|
||||
if fallocSize > 0 {
|
||||
if err = fsFAllocate(int(writer.Fd()), 0, fallocSize); err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -300,47 +284,18 @@ func fsCreateFile(filePath string, reader io.Reader, buf []byte, fallocSize int6
|
||||
if buf != nil {
|
||||
bytesWritten, err = io.CopyBuffer(writer, reader, buf)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
} else {
|
||||
bytesWritten, err = io.Copy(writer, reader)
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
}
|
||||
|
||||
return bytesWritten, nil
|
||||
}
|
||||
|
||||
// Removes uploadID at destination path.
|
||||
func fsRemoveUploadIDPath(basePath, uploadIDPath string) error {
|
||||
if basePath == "" || uploadIDPath == "" {
|
||||
return traceError(errInvalidArgument)
|
||||
}
|
||||
if err := checkPathLength(basePath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
if err := checkPathLength(uploadIDPath); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
// List all the entries in uploadID.
|
||||
entries, err := readDir(uploadIDPath)
|
||||
if err != nil && err != errFileNotFound {
|
||||
return traceError(err)
|
||||
}
|
||||
|
||||
// Delete all the entries obtained from previous readdir.
|
||||
for _, entryPath := range entries {
|
||||
err = fsDeleteFile(basePath, pathJoin(uploadIDPath, entryPath))
|
||||
if err != nil && err != errFileNotFound {
|
||||
return traceError(err)
|
||||
}
|
||||
}
|
||||
|
||||
fsRemoveDir(uploadIDPath)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fsFAllocate is similar to Fallocate but provides a convenient
|
||||
// wrapper to handle various operating system specific errors.
|
||||
func fsFAllocate(fd int, offset int64, len int64) (err error) {
|
||||
@@ -367,35 +322,32 @@ func fsFAllocate(fd int, offset int64, len int64) (err error) {
|
||||
// missing parents if they don't exist.
|
||||
func fsRenameFile(sourcePath, destPath string) error {
|
||||
if err := checkPathLength(sourcePath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
if err := checkPathLength(destPath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
// Verify if source path exists.
|
||||
if _, err := os.Stat((sourcePath)); err != nil {
|
||||
if _, err := os.Stat(sourcePath); err != nil {
|
||||
return osErrToFSFileErr(err)
|
||||
}
|
||||
if err := os.MkdirAll(pathutil.Dir(destPath), 0777); err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
if err := os.Rename((sourcePath), (destPath)); err != nil {
|
||||
if isSysErrCrossDevice(err) {
|
||||
return traceError(fmt.Errorf("%s (%s)->(%s)", errCrossDeviceLink, sourcePath, destPath))
|
||||
}
|
||||
return traceError(err)
|
||||
|
||||
if err := renameAll(sourcePath, destPath); err != nil {
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fsDeleteFile is a wrapper for deleteFile(), after checking the path length.
|
||||
func fsDeleteFile(basePath, deletePath string) error {
|
||||
if err := checkPathLength(basePath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
if err := checkPathLength(deletePath); err != nil {
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
return deleteFile(basePath, deletePath)
|
||||
|
||||
+19
-40
@@ -24,31 +24,10 @@ import (
|
||||
"path"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
)
|
||||
|
||||
// Tests - fsMkdirAll()
|
||||
func TestFSMkdirAll(t *testing.T) {
|
||||
// create posix test setup
|
||||
_, path, err := newPosixTestSetup()
|
||||
if err != nil {
|
||||
t.Fatalf("Unable to create posix test setup, %s", err)
|
||||
}
|
||||
defer os.RemoveAll(path)
|
||||
|
||||
if err = fsMkdirAll(""); errorCause(err) != errInvalidArgument {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if err = fsMkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if err = fsMkdirAll(pathJoin(path, "success-vol", "success-object")); err != nil {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSRenameFile(t *testing.T) {
|
||||
// create posix test setup
|
||||
_, path, err := newPosixTestSetup()
|
||||
@@ -63,13 +42,13 @@ func TestFSRenameFile(t *testing.T) {
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNotFound {
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNotFound {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errorCause(err) != errFileNameTooLong {
|
||||
if err = fsRenameFile(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); errors.Cause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
|
||||
if err = fsRenameFile(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
}
|
||||
@@ -84,11 +63,11 @@ func TestFSStats(t *testing.T) {
|
||||
|
||||
// Setup test environment.
|
||||
|
||||
if err = fsMkdir(""); errorCause(err) != errInvalidArgument {
|
||||
if err = fsMkdir(""); errors.Cause(err) != errInvalidArgument {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if err = fsMkdir(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errorCause(err) != errFileNameTooLong {
|
||||
if err = fsMkdir(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); errors.Cause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
@@ -103,7 +82,7 @@ func TestFSStats(t *testing.T) {
|
||||
// Seek back.
|
||||
reader.Seek(0, 0)
|
||||
|
||||
if err = fsMkdir(pathJoin(path, "success-vol", "success-file")); errorCause(err) != errVolumeExists {
|
||||
if err = fsMkdir(pathJoin(path, "success-vol", "success-file")); errors.Cause(err) != errVolumeExists {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
@@ -191,11 +170,11 @@ func TestFSStats(t *testing.T) {
|
||||
for i, testCase := range testCases {
|
||||
if testCase.srcPath != "" {
|
||||
if _, err := fsStatFile(pathJoin(testCase.srcFSPath, testCase.srcVol,
|
||||
testCase.srcPath)); errorCause(err) != testCase.expectedErr {
|
||||
testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
|
||||
t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
} else {
|
||||
if _, err := fsStatVolume(pathJoin(testCase.srcFSPath, testCase.srcVol)); errorCause(err) != testCase.expectedErr {
|
||||
if _, err := fsStatVolume(pathJoin(testCase.srcFSPath, testCase.srcVol)); errors.Cause(err) != testCase.expectedErr {
|
||||
t.Fatalf("TestPosix case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
@@ -214,11 +193,11 @@ func TestFSCreateAndOpen(t *testing.T) {
|
||||
t.Fatalf("Unable to create directory, %s", err)
|
||||
}
|
||||
|
||||
if _, err = fsCreateFile("", nil, nil, 0); errorCause(err) != errInvalidArgument {
|
||||
if _, err = fsCreateFile("", nil, nil, 0); errors.Cause(err) != errInvalidArgument {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if _, _, err = fsOpenFile("", -1); errorCause(err) != errInvalidArgument {
|
||||
if _, _, err = fsOpenFile("", -1); errors.Cause(err) != errInvalidArgument {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
@@ -252,17 +231,17 @@ func TestFSCreateAndOpen(t *testing.T) {
|
||||
|
||||
for i, testCase := range testCases {
|
||||
_, err = fsCreateFile(pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
|
||||
if errorCause(err) != testCase.expectedErr {
|
||||
if errors.Cause(err) != testCase.expectedErr {
|
||||
t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
_, _, err = fsOpenFile(pathJoin(path, testCase.srcVol, testCase.srcPath), 0)
|
||||
if errorCause(err) != testCase.expectedErr {
|
||||
if errors.Cause(err) != testCase.expectedErr {
|
||||
t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to open a directory.
|
||||
if _, _, err = fsOpenFile(pathJoin(path), 0); errorCause(err) != errIsNotRegular {
|
||||
if _, _, err = fsOpenFile(pathJoin(path), 0); errors.Cause(err) != errIsNotRegular {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
}
|
||||
@@ -364,7 +343,7 @@ func TestFSDeletes(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
|
||||
if err = fsDeleteFile(testCase.basePath, pathJoin(testCase.basePath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
|
||||
t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
}
|
||||
@@ -498,11 +477,11 @@ func TestFSRemoves(t *testing.T) {
|
||||
|
||||
for i, testCase := range testCases {
|
||||
if testCase.srcPath != "" {
|
||||
if err = fsRemoveFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
|
||||
if err = fsRemoveFile(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
|
||||
t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
} else {
|
||||
if err = fsRemoveDir(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errorCause(err) != testCase.expectedErr {
|
||||
if err = fsRemoveDir(pathJoin(testCase.srcFSPath, testCase.srcVol, testCase.srcPath)); errors.Cause(err) != testCase.expectedErr {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
@@ -512,11 +491,11 @@ func TestFSRemoves(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err = fsRemoveAll(""); errorCause(err) != errInvalidArgument {
|
||||
if err = fsRemoveAll(""); errors.Cause(err) != errInvalidArgument {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if err = fsRemoveAll("my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); errorCause(err) != errFileNameTooLong {
|
||||
if err = fsRemoveAll("my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"); errors.Cause(err) != errFileNameTooLong {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
+12
-182
@@ -17,15 +17,13 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
pathutil "path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/mimedb"
|
||||
"github.com/tidwall/gjson"
|
||||
@@ -121,64 +119,21 @@ func (m fsMetaV1) ToObjectInfo(bucket, object string, fi os.FileInfo) ObjectInfo
|
||||
// etag/md5Sum has already been extracted. We need to
|
||||
// remove to avoid it from appearing as part of
|
||||
// response headers. e.g, X-Minio-* or X-Amz-*.
|
||||
objInfo.UserDefined = cleanMetaETag(m.Meta)
|
||||
objInfo.UserDefined = cleanMetadata(m.Meta)
|
||||
|
||||
// Success..
|
||||
return objInfo
|
||||
}
|
||||
|
||||
// ObjectPartIndex - returns the index of matching object part number.
|
||||
func (m fsMetaV1) ObjectPartIndex(partNumber int) (partIndex int) {
|
||||
for i, part := range m.Parts {
|
||||
if partNumber == part.Number {
|
||||
partIndex = i
|
||||
return partIndex
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// AddObjectPart - add a new object part in order.
|
||||
func (m *fsMetaV1) AddObjectPart(partNumber int, partName string, partETag string, partSize int64) {
|
||||
partInfo := objectPartInfo{
|
||||
Number: partNumber,
|
||||
Name: partName,
|
||||
ETag: partETag,
|
||||
Size: partSize,
|
||||
}
|
||||
|
||||
// Update part info if it already exists.
|
||||
for i, part := range m.Parts {
|
||||
if partNumber == part.Number {
|
||||
m.Parts[i] = partInfo
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Proceed to include new part info.
|
||||
m.Parts = append(m.Parts, partInfo)
|
||||
|
||||
// Parts in fsMeta should be in sorted order by part number.
|
||||
sort.Sort(byObjectPartNumber(m.Parts))
|
||||
}
|
||||
|
||||
func (m *fsMetaV1) WriteTo(lk *lock.LockedFile) (n int64, err error) {
|
||||
var metadataBytes []byte
|
||||
metadataBytes, err = json.Marshal(m)
|
||||
if err = jsonSave(lk, m); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
fi, err := lk.Stat()
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err = lk.Truncate(0); err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
|
||||
if _, err = lk.Write(metadataBytes); err != nil {
|
||||
return 0, traceError(err)
|
||||
}
|
||||
|
||||
// Success.
|
||||
return int64(len(metadataBytes)), nil
|
||||
return fi.Size(), nil
|
||||
}
|
||||
|
||||
func parseFSVersion(fsMetaBuf []byte) string {
|
||||
@@ -203,35 +158,20 @@ func parseFSMetaMap(fsMetaBuf []byte) map[string]string {
|
||||
return metaMap
|
||||
}
|
||||
|
||||
func parseFSParts(fsMetaBuf []byte) []objectPartInfo {
|
||||
// Parse the FS Parts.
|
||||
partsResult := gjson.GetBytes(fsMetaBuf, "parts").Array()
|
||||
partInfo := make([]objectPartInfo, len(partsResult))
|
||||
for i, p := range partsResult {
|
||||
info := objectPartInfo{}
|
||||
info.Number = int(p.Get("number").Int())
|
||||
info.Name = p.Get("name").String()
|
||||
info.ETag = p.Get("etag").String()
|
||||
info.Size = p.Get("size").Int()
|
||||
partInfo[i] = info
|
||||
}
|
||||
return partInfo
|
||||
}
|
||||
|
||||
func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
|
||||
var fsMetaBuf []byte
|
||||
fi, err := lk.Stat()
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
fsMetaBuf, err = ioutil.ReadAll(io.NewSectionReader(lk, 0, fi.Size()))
|
||||
if err != nil {
|
||||
return 0, traceError(err)
|
||||
return 0, errors.Trace(err)
|
||||
}
|
||||
|
||||
if len(fsMetaBuf) == 0 {
|
||||
return 0, traceError(io.EOF)
|
||||
return 0, errors.Trace(io.EOF)
|
||||
}
|
||||
|
||||
// obtain version.
|
||||
@@ -243,15 +183,12 @@ func (m *fsMetaV1) ReadFrom(lk *lock.LockedFile) (n int64, err error) {
|
||||
// Verify if the format is valid, return corrupted format
|
||||
// for unrecognized formats.
|
||||
if !isFSMetaValid(m.Version, m.Format) {
|
||||
return 0, traceError(errCorruptedFormat)
|
||||
return 0, errors.Trace(errCorruptedFormat)
|
||||
}
|
||||
|
||||
// obtain metadata.
|
||||
m.Meta = parseFSMetaMap(fsMetaBuf)
|
||||
|
||||
// obtain parts info list.
|
||||
m.Parts = parseFSParts(fsMetaBuf)
|
||||
|
||||
// obtain minio release date.
|
||||
m.Minio.Release = parseFSRelease(fsMetaBuf)
|
||||
|
||||
@@ -267,110 +204,3 @@ func newFSMetaV1() (fsMeta fsMetaV1) {
|
||||
fsMeta.Minio.Release = ReleaseTag
|
||||
return fsMeta
|
||||
}
|
||||
|
||||
// Check if disk has already a valid format, holds a read lock and
|
||||
// upon success returns it to the caller to be closed.
|
||||
func checkLockedValidFormatFS(fsPath string) (*lock.RLockedFile, error) {
|
||||
fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
|
||||
|
||||
rlk, err := lock.RLockedOpenFile((fsFormatPath))
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
// If format.json not found then
|
||||
// its an unformatted disk.
|
||||
return nil, traceError(errUnformattedDisk)
|
||||
}
|
||||
return nil, traceError(err)
|
||||
}
|
||||
|
||||
var format = &formatConfigV1{}
|
||||
if err = format.LoadFormat(rlk.LockedFile); err != nil {
|
||||
rlk.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Check format FS.
|
||||
if err = format.CheckFS(); err != nil {
|
||||
rlk.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Always return read lock here and should be closed by the caller.
|
||||
return rlk, traceError(err)
|
||||
}
|
||||
|
||||
// Creates a new format.json if unformatted.
|
||||
func createFormatFS(fsPath string) error {
|
||||
fsFormatPath := pathJoin(fsPath, minioMetaBucket, formatConfigFile)
|
||||
|
||||
// Attempt a write lock on formatConfigFile `format.json`
|
||||
// file stored in minioMetaBucket(.minio.sys) directory.
|
||||
lk, err := lock.TryLockedOpenFile((fsFormatPath), os.O_RDWR|os.O_CREATE, 0600)
|
||||
if err != nil {
|
||||
return traceError(err)
|
||||
}
|
||||
// Close the locked file upon return.
|
||||
defer lk.Close()
|
||||
|
||||
// Load format on disk, checks if we are unformatted
|
||||
// writes the new format.json
|
||||
var format = &formatConfigV1{}
|
||||
err = format.LoadFormat(lk)
|
||||
if errorCause(err) == errUnformattedDisk {
|
||||
_, err = newFSFormat().WriteTo(lk)
|
||||
return err
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func initFormatFS(fsPath string) (rlk *lock.RLockedFile, err error) {
|
||||
// This loop validates format.json by holding a read lock and
|
||||
// proceeds if disk unformatted to hold non-blocking WriteLock
|
||||
// If for some reason non-blocking WriteLock fails and the error
|
||||
// is lock.ErrAlreadyLocked i.e some other process is holding a
|
||||
// lock we retry in the loop again.
|
||||
for {
|
||||
// Validate the `format.json` for expected values.
|
||||
rlk, err = checkLockedValidFormatFS(fsPath)
|
||||
switch {
|
||||
case err == nil:
|
||||
// Holding a read lock ensures that any write lock operation
|
||||
// is blocked if attempted in-turn avoiding corruption on
|
||||
// the backend disk.
|
||||
return rlk, nil
|
||||
case errorCause(err) == errUnformattedDisk:
|
||||
if err = createFormatFS(fsPath); err != nil {
|
||||
// Existing write locks detected.
|
||||
if errorCause(err) == lock.ErrAlreadyLocked {
|
||||
// Lock already present, sleep and attempt again.
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
continue
|
||||
}
|
||||
|
||||
// Unexpected error, return.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Loop will continue to attempt a read-lock on `format.json`.
|
||||
default:
|
||||
// Unhandled error return.
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Return if the part info in uploadedParts and CompleteParts are same.
|
||||
func isPartsSame(uploadedParts []objectPartInfo, CompleteParts []CompletePart) bool {
|
||||
if len(uploadedParts) != len(CompleteParts) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range CompleteParts {
|
||||
if uploadedParts[i].Number != CompleteParts[i].PartNumber ||
|
||||
uploadedParts[i].ETag != CompleteParts[i].ETag {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
+384
-785
File diff suppressed because it is too large
Load Diff
+12
-87
@@ -22,8 +22,11 @@ import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// Tests cleanup multipart uploads for filesystem backend.
|
||||
func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
@@ -56,78 +59,13 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
|
||||
|
||||
// Check if upload id was already purged.
|
||||
if err = obj.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
|
||||
err = errorCause(err)
|
||||
err = errors.Cause(err)
|
||||
if _, ok := err.(InvalidUploadID); !ok {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Tests cleanup of stale upload ids.
|
||||
func TestFSCleanupMultipartUpload(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer os.RemoveAll(disk)
|
||||
|
||||
obj := initFSObjects(disk, t)
|
||||
fs := obj.(*fsObjects)
|
||||
|
||||
// Close the multipart cleanup go-routine.
|
||||
// In this test we are going to manually call
|
||||
// the function which actually cleans the stale
|
||||
// uploads.
|
||||
globalServiceDoneCh <- struct{}{}
|
||||
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
uploadID, err := obj.NewMultipartUpload(bucketName, objectName, nil)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
if err = fs.cleanupStaleMultipartUpload(bucketName, 0); err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
// Check if upload id was already purged.
|
||||
if err = obj.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
|
||||
err = errorCause(err)
|
||||
if _, ok := err.(InvalidUploadID); !ok {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestFSWriteUploadJSON - tests for writeUploadJSON for FS
|
||||
func TestFSWriteUploadJSON(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
defer os.RemoveAll(disk)
|
||||
|
||||
obj := initFSObjects(disk, t)
|
||||
fs := obj.(*fsObjects)
|
||||
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
|
||||
obj.MakeBucketWithLocation(bucketName, "")
|
||||
_, err := obj.NewMultipartUpload(bucketName, objectName, nil)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
|
||||
// newMultipartUpload will fail.
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
_, err = obj.NewMultipartUpload(bucketName, objectName, nil)
|
||||
if err != nil {
|
||||
if _, ok := errorCause(err).(BucketNotFound); !ok {
|
||||
t.Fatal("Unexpected err: ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewMultipartUploadFaultyDisk - test NewMultipartUpload with faulty disks
|
||||
func TestNewMultipartUploadFaultyDisk(t *testing.T) {
|
||||
// Prepare for tests
|
||||
@@ -146,7 +84,7 @@ func TestNewMultipartUploadFaultyDisk(t *testing.T) {
|
||||
// Test with disk removed.
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
if _, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"}); err != nil {
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
}
|
||||
@@ -184,7 +122,7 @@ func TestPutObjectPartFaultyDisk(t *testing.T) {
|
||||
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
_, err = fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum))
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
}
|
||||
@@ -212,21 +150,16 @@ func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
|
||||
|
||||
md5Hex := getMD5Hash(data)
|
||||
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
|
||||
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
if _, err := fs.CompleteMultipartUpload(bucketName, objectName, uploadID, parts); err != nil {
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
|
||||
// TestCompleteMultipartUpload - test CompleteMultipartUpload
|
||||
func TestCompleteMultipartUpload(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
@@ -260,7 +193,7 @@ func TestCompleteMultipartUpload(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
|
||||
// TestCompleteMultipartUpload - test CompleteMultipartUpload
|
||||
func TestAbortMultipartUpload(t *testing.T) {
|
||||
// Prepare for tests
|
||||
disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
@@ -286,7 +219,7 @@ func TestAbortMultipartUpload(t *testing.T) {
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, "")); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
|
||||
if err := fs.AbortMultipartUpload(bucketName, objectName, uploadID); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
@@ -303,27 +236,19 @@ func TestListMultipartUploadsFaultyDisk(t *testing.T) {
|
||||
fs := obj.(*fsObjects)
|
||||
bucketName := "bucket"
|
||||
objectName := "object"
|
||||
data := []byte("12345")
|
||||
|
||||
if err := obj.MakeBucketWithLocation(bucketName, ""); err != nil {
|
||||
t.Fatal("Cannot create bucket, err: ", err)
|
||||
}
|
||||
|
||||
uploadID, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
|
||||
_, err := fs.NewMultipartUpload(bucketName, objectName, map[string]string{"X-Amz-Meta-xid": "3f"})
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
md5Hex := getMD5Hash(data)
|
||||
sha256sum := ""
|
||||
|
||||
if _, err := fs.PutObjectPart(bucketName, objectName, uploadID, 1, mustGetHashReader(t, bytes.NewReader(data), 5, md5Hex, sha256sum)); err != nil {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
if _, err := fs.ListMultipartUploads(bucketName, objectName, "", "", "", 1000); err != nil {
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error ", err)
|
||||
}
|
||||
}
|
||||
|
||||
+173
-96
@@ -25,10 +25,14 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/lock"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
// fsObjects - Implements fs object layer.
|
||||
@@ -49,8 +53,18 @@ type fsObjects struct {
|
||||
// ListObjects pool management.
|
||||
listPool *treeWalkPool
|
||||
|
||||
// To manage the appendRoutine go0routines
|
||||
bgAppend *backgroundAppend
|
||||
appendFileMap map[string]*fsAppendFile
|
||||
appendFileMapMu sync.Mutex
|
||||
|
||||
// To manage the appendRoutine go-routines
|
||||
nsMutex *nsLockMap
|
||||
}
|
||||
|
||||
// Represents the background append file.
|
||||
type fsAppendFile struct {
|
||||
sync.Mutex
|
||||
parts []PartInfo // List of parts appended.
|
||||
filePath string // Absolute path of the file in the temp location.
|
||||
}
|
||||
|
||||
// Initializes meta volume on all the fs path.
|
||||
@@ -133,10 +147,9 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
rwPool: &fsIOPool{
|
||||
readersMap: make(map[string]*lock.RLockedFile),
|
||||
},
|
||||
listPool: newTreeWalkPool(globalLookupTimeout),
|
||||
bgAppend: &backgroundAppend{
|
||||
infoMap: make(map[string]bgAppendPartsInfo),
|
||||
},
|
||||
nsMutex: newNSLock(false),
|
||||
listPool: newTreeWalkPool(globalLookupTimeout),
|
||||
appendFileMap: make(map[string]*fsAppendFile),
|
||||
}
|
||||
|
||||
// Once the filesystem has initialized hold the read lock for
|
||||
@@ -155,15 +168,13 @@ func newFSObjectLayer(fsPath string) (ObjectLayer, error) {
|
||||
return nil, fmt.Errorf("Unable to initialize event notification. %s", err)
|
||||
}
|
||||
|
||||
// Start background process to cleanup old files in `.minio.sys`.
|
||||
go fs.cleanupStaleMultipartUploads(fsMultipartCleanupInterval, fsMultipartExpiry, globalServiceDoneCh)
|
||||
|
||||
go fs.cleanupStaleMultipartUploads(multipartCleanupInterval, multipartExpiry, globalServiceDoneCh)
|
||||
// Return successfully initialized object layer.
|
||||
return fs, nil
|
||||
}
|
||||
|
||||
// Should be called when process shuts down.
|
||||
func (fs fsObjects) Shutdown() error {
|
||||
func (fs *fsObjects) Shutdown() error {
|
||||
fs.fsFormatRlk.Close()
|
||||
|
||||
// Cleanup and delete tmp uuid.
|
||||
@@ -171,7 +182,7 @@ func (fs fsObjects) Shutdown() error {
|
||||
}
|
||||
|
||||
// StorageInfo - returns underlying storage statistics.
|
||||
func (fs fsObjects) StorageInfo() StorageInfo {
|
||||
func (fs *fsObjects) StorageInfo() StorageInfo {
|
||||
info, err := getDiskInfo((fs.fsPath))
|
||||
errorIf(err, "Unable to get disk info %#v", fs.fsPath)
|
||||
storageInfo := StorageInfo{
|
||||
@@ -182,22 +193,34 @@ func (fs fsObjects) StorageInfo() StorageInfo {
|
||||
return storageInfo
|
||||
}
|
||||
|
||||
// Locking operations
|
||||
|
||||
// List namespace locks held in object layer
|
||||
func (fs *fsObjects) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
|
||||
return []VolumeLockInfo{}, NotImplemented{}
|
||||
}
|
||||
|
||||
// Clear namespace locks held in object layer
|
||||
func (fs *fsObjects) ClearLocks([]VolumeLockInfo) error {
|
||||
return NotImplemented{}
|
||||
}
|
||||
|
||||
/// Bucket operations
|
||||
|
||||
// getBucketDir - will convert incoming bucket names to
|
||||
// corresponding valid bucket names on the backend in a platform
|
||||
// compatible way for all operating systems.
|
||||
func (fs fsObjects) getBucketDir(bucket string) (string, error) {
|
||||
func (fs *fsObjects) getBucketDir(bucket string) (string, error) {
|
||||
// Verify if bucket is valid.
|
||||
if !IsValidBucketName(bucket) {
|
||||
return "", traceError(BucketNameInvalid{Bucket: bucket})
|
||||
return "", errors.Trace(BucketNameInvalid{Bucket: bucket})
|
||||
}
|
||||
|
||||
bucketDir := pathJoin(fs.fsPath, bucket)
|
||||
return bucketDir, nil
|
||||
}
|
||||
|
||||
func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
|
||||
func (fs *fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
|
||||
bucketDir, err := fs.getBucketDir(bucket)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -211,7 +234,12 @@ func (fs fsObjects) statBucketDir(bucket string) (os.FileInfo, error) {
|
||||
|
||||
// MakeBucket - create a new bucket, returns if it
|
||||
// already exists.
|
||||
func (fs fsObjects) MakeBucketWithLocation(bucket, location string) error {
|
||||
func (fs *fsObjects) MakeBucketWithLocation(bucket, location string) error {
|
||||
bucketLock := fs.nsMutex.NewNSLock(bucket, "")
|
||||
if err := bucketLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer bucketLock.Unlock()
|
||||
bucketDir, err := fs.getBucketDir(bucket)
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
@@ -225,7 +253,7 @@ func (fs fsObjects) MakeBucketWithLocation(bucket, location string) error {
|
||||
}
|
||||
|
||||
// GetBucketInfo - fetch bucket metadata info.
|
||||
func (fs fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
func (fs *fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
st, err := fs.statBucketDir(bucket)
|
||||
if err != nil {
|
||||
return bi, toObjectErr(err, bucket)
|
||||
@@ -240,14 +268,14 @@ func (fs fsObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
}
|
||||
|
||||
// ListBuckets - list all s3 compatible buckets (directories) at fsPath.
|
||||
func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
|
||||
func (fs *fsObjects) ListBuckets() ([]BucketInfo, error) {
|
||||
if err := checkPathLength(fs.fsPath); err != nil {
|
||||
return nil, traceError(err)
|
||||
return nil, errors.Trace(err)
|
||||
}
|
||||
var bucketInfos []BucketInfo
|
||||
entries, err := readDir((fs.fsPath))
|
||||
if err != nil {
|
||||
return nil, toObjectErr(traceError(errDiskNotFound))
|
||||
return nil, toObjectErr(errors.Trace(errDiskNotFound))
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
@@ -281,7 +309,7 @@ func (fs fsObjects) ListBuckets() ([]BucketInfo, error) {
|
||||
|
||||
// DeleteBucket - delete a bucket and all the metadata associated
|
||||
// with the bucket including pending multipart, object metadata.
|
||||
func (fs fsObjects) DeleteBucket(bucket string) error {
|
||||
func (fs *fsObjects) DeleteBucket(bucket string) error {
|
||||
bucketDir, err := fs.getBucketDir(bucket)
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
@@ -312,7 +340,29 @@ func (fs fsObjects) DeleteBucket(bucket string) error {
|
||||
// CopyObject - copy object source object to destination object.
|
||||
// if source object and destination object are same we only
|
||||
// update metadata.
|
||||
func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string) (oi ObjectInfo, e error) {
|
||||
func (fs *fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string, metadata map[string]string, srcEtag string) (oi ObjectInfo, e error) {
|
||||
cpSrcDstSame := srcBucket == dstBucket && srcObject == dstObject
|
||||
// Hold write lock on destination since in both cases
|
||||
// - if source and destination are same
|
||||
// - if source and destination are different
|
||||
// it is the sole mutating state.
|
||||
objectDWLock := fs.nsMutex.NewNSLock(dstBucket, dstObject)
|
||||
if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
defer objectDWLock.Unlock()
|
||||
// if source and destination are different, we have to hold
|
||||
// additional read lock as well to protect against writes on
|
||||
// source.
|
||||
if !cpSrcDstSame {
|
||||
// Hold read locks on source object only if we are
|
||||
// going to read data from source object.
|
||||
objectSRLock := fs.nsMutex.NewNSLock(srcBucket, srcObject)
|
||||
if err := objectSRLock.GetRLock(globalObjectTimeout); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
defer objectSRLock.RUnlock()
|
||||
}
|
||||
if _, err := fs.statBucketDir(srcBucket); err != nil {
|
||||
return oi, toObjectErr(err, srcBucket)
|
||||
}
|
||||
@@ -322,6 +372,15 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, srcBucket, srcObject)
|
||||
}
|
||||
if srcEtag != "" {
|
||||
etag, perr := fs.getObjectETag(srcBucket, srcObject)
|
||||
if perr != nil {
|
||||
return oi, toObjectErr(perr, srcBucket, srcObject)
|
||||
}
|
||||
if etag != srcEtag {
|
||||
return oi, toObjectErr(errors.Trace(InvalidETag{}), srcBucket, srcObject)
|
||||
}
|
||||
}
|
||||
|
||||
// Check if this request is only metadata update.
|
||||
cpMetadataOnly := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
|
||||
@@ -330,7 +389,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
|
||||
var wlk *lock.LockedFile
|
||||
wlk, err = fs.rwPool.Write(fsMetaPath)
|
||||
if err != nil {
|
||||
return oi, toObjectErr(traceError(err), srcBucket, srcObject)
|
||||
return oi, toObjectErr(errors.Trace(err), srcBucket, srcObject)
|
||||
}
|
||||
// This close will allow for locks to be synchronized on `fs.json`.
|
||||
defer wlk.Close()
|
||||
@@ -354,7 +413,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
|
||||
|
||||
go func() {
|
||||
var startOffset int64 // Read the whole file.
|
||||
if gerr := fs.GetObject(srcBucket, srcObject, startOffset, length, pipeWriter); gerr != nil {
|
||||
if gerr := fs.getObject(srcBucket, srcObject, startOffset, length, pipeWriter, ""); gerr != nil {
|
||||
errorIf(gerr, "Unable to read %s/%s.", srcBucket, srcObject)
|
||||
pipeWriter.CloseWithError(gerr)
|
||||
return
|
||||
@@ -367,7 +426,7 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
|
||||
return oi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
|
||||
objInfo, err := fs.PutObject(dstBucket, dstObject, hashReader, metadata)
|
||||
objInfo, err := fs.putObject(dstBucket, dstObject, hashReader, metadata)
|
||||
if err != nil {
|
||||
return oi, toObjectErr(err, dstBucket, dstObject)
|
||||
}
|
||||
@@ -384,40 +443,60 @@ func (fs fsObjects) CopyObject(srcBucket, srcObject, dstBucket, dstObject string
|
||||
//
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer) (err error) {
|
||||
if err = checkBucketAndObjectNamesFS(bucket, object); err != nil {
|
||||
func (fs *fsObjects) GetObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) {
|
||||
if err = checkGetObjArgs(bucket, object); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Lock the object before reading.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objectLock.RUnlock()
|
||||
return fs.getObject(bucket, object, offset, length, writer, etag)
|
||||
}
|
||||
|
||||
// getObject - wrapper for GetObject
|
||||
func (fs *fsObjects) getObject(bucket, object string, offset int64, length int64, writer io.Writer, etag string) (err error) {
|
||||
if _, err = fs.statBucketDir(bucket); err != nil {
|
||||
return toObjectErr(err, bucket)
|
||||
}
|
||||
|
||||
// Offset cannot be negative.
|
||||
if offset < 0 {
|
||||
return toObjectErr(traceError(errUnexpected), bucket, object)
|
||||
return toObjectErr(errors.Trace(errUnexpected), bucket, object)
|
||||
}
|
||||
|
||||
// Writer cannot be nil.
|
||||
if writer == nil {
|
||||
return toObjectErr(traceError(errUnexpected), bucket, object)
|
||||
return toObjectErr(errors.Trace(errUnexpected), bucket, object)
|
||||
}
|
||||
|
||||
// If its a directory request, we return an empty body.
|
||||
if hasSuffix(object, slashSeparator) {
|
||||
_, err = writer.Write([]byte(""))
|
||||
return toObjectErr(traceError(err), bucket, object)
|
||||
return toObjectErr(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
if bucket != minioMetaBucket {
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fsMetaJSONFile)
|
||||
_, err = fs.rwPool.Open(fsMetaPath)
|
||||
if err != nil && err != errFileNotFound {
|
||||
return toObjectErr(traceError(err), bucket, object)
|
||||
return toObjectErr(errors.Trace(err), bucket, object)
|
||||
}
|
||||
defer fs.rwPool.Close(fsMetaPath)
|
||||
}
|
||||
|
||||
if etag != "" {
|
||||
objEtag, perr := fs.getObjectETag(bucket, object)
|
||||
if perr != nil {
|
||||
return toObjectErr(errors.Trace(perr), bucket, object)
|
||||
}
|
||||
if objEtag != etag {
|
||||
return toObjectErr(errors.Trace(InvalidETag{}), bucket, object)
|
||||
}
|
||||
}
|
||||
// Read the object, doesn't exist returns an s3 compatible error.
|
||||
fsObjPath := pathJoin(fs.fsPath, bucket, object)
|
||||
reader, size, err := fsOpenFile(fsObjPath, offset)
|
||||
@@ -438,7 +517,7 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
|
||||
|
||||
// Reply back invalid range if the input offset and length fall out of range.
|
||||
if offset > size || offset+length > size {
|
||||
return traceError(InvalidRange{offset, length, size})
|
||||
return errors.Trace(InvalidRange{offset, length, size})
|
||||
}
|
||||
|
||||
// Allocate a staging buffer.
|
||||
@@ -446,14 +525,14 @@ func (fs fsObjects) GetObject(bucket, object string, offset int64, length int64,
|
||||
|
||||
_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
|
||||
|
||||
return toObjectErr(traceError(err), bucket, object)
|
||||
return toObjectErr(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
|
||||
func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
|
||||
func (fs *fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
|
||||
fsMeta := fsMetaV1{}
|
||||
fi, err := fsStatDir(pathJoin(fs.fsPath, bucket, object))
|
||||
if err != nil && errorCause(err) != errFileAccessDenied {
|
||||
if err != nil && errors.Cause(err) != errFileAccessDenied {
|
||||
return oi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
if fi != nil {
|
||||
@@ -477,7 +556,7 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
|
||||
// `fs.json` can be empty due to previously failed
|
||||
// PutObject() transaction, if we arrive at such
|
||||
// a situation we just ignore and continue.
|
||||
if errorCause(rerr) != io.EOF {
|
||||
if errors.Cause(rerr) != io.EOF {
|
||||
return oi, toObjectErr(rerr, bucket, object)
|
||||
}
|
||||
}
|
||||
@@ -485,7 +564,7 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
|
||||
|
||||
// Ignore if `fs.json` is not available, this is true for pre-existing data.
|
||||
if err != nil && err != errFileNotFound {
|
||||
return oi, toObjectErr(traceError(err), bucket, object)
|
||||
return oi, toObjectErr(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
// Stat the file to get file size.
|
||||
@@ -497,25 +576,16 @@ func (fs fsObjects) getObjectInfo(bucket, object string) (oi ObjectInfo, e error
|
||||
return fsMeta.ToObjectInfo(bucket, object, fi), nil
|
||||
}
|
||||
|
||||
// Checks bucket and object name validity, returns nil if both are valid.
|
||||
func checkBucketAndObjectNamesFS(bucket, object string) error {
|
||||
// Verify if bucket is valid.
|
||||
if !IsValidBucketName(bucket) {
|
||||
return traceError(BucketNameInvalid{Bucket: bucket})
|
||||
}
|
||||
// Verify if object is valid.
|
||||
if len(object) == 0 {
|
||||
return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
|
||||
}
|
||||
if !IsValidObjectPrefix(object) {
|
||||
return traceError(ObjectNameInvalid{Bucket: bucket, Object: object})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetObjectInfo - reads object metadata and replies back ObjectInfo.
|
||||
func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
|
||||
if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
|
||||
func (fs *fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error) {
|
||||
// Lock the object before reading.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
if err := objectLock.GetRLock(globalObjectTimeout); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
defer objectLock.RUnlock()
|
||||
|
||||
if err := checkGetObjArgs(bucket, object); err != nil {
|
||||
return oi, err
|
||||
}
|
||||
|
||||
@@ -529,7 +599,7 @@ func (fs fsObjects) GetObjectInfo(bucket, object string) (oi ObjectInfo, e error
|
||||
// This function does the following check, suppose
|
||||
// object is "a/b/c/d", stat makes sure that objects ""a/b/c""
|
||||
// "a/b" and "a" do not exist.
|
||||
func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
|
||||
func (fs *fsObjects) parentDirIsObject(bucket, parent string) bool {
|
||||
var isParentDirObject func(string) bool
|
||||
isParentDirObject = func(p string) bool {
|
||||
if p == "." || p == "/" {
|
||||
@@ -550,12 +620,22 @@ func (fs fsObjects) parentDirIsObject(bucket, parent string) bool {
|
||||
// until EOF, writes data directly to configured filesystem path.
|
||||
// Additionally writes `fs.json` which carries the necessary metadata
|
||||
// for future object operations.
|
||||
func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
|
||||
func (fs *fsObjects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
|
||||
// Lock the object.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
if err := objectLock.GetLock(globalObjectTimeout); err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
defer objectLock.Unlock()
|
||||
return fs.putObject(bucket, object, data, metadata)
|
||||
}
|
||||
|
||||
// putObject - wrapper for PutObject
|
||||
func (fs *fsObjects) putObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, retErr error) {
|
||||
// No metadata is set, allocate a new one.
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string)
|
||||
}
|
||||
|
||||
var err error
|
||||
|
||||
// Validate if bucket name is valid and exists.
|
||||
@@ -572,9 +652,9 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
|
||||
if isObjectDir(object, data.Size()) {
|
||||
// Check if an object is present as one of the parent dir.
|
||||
if fs.parentDirIsObject(bucket, path.Dir(object)) {
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
|
||||
return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object)
|
||||
}
|
||||
if err = fsMkdirAll(pathJoin(fs.fsPath, bucket, object)); err != nil {
|
||||
if err = mkdirAll(pathJoin(fs.fsPath, bucket, object), 0777); err != nil {
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
var fi os.FileInfo
|
||||
@@ -590,12 +670,12 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
|
||||
|
||||
// Check if an object is present as one of the parent dir.
|
||||
if fs.parentDirIsObject(bucket, path.Dir(object)) {
|
||||
return ObjectInfo{}, toObjectErr(traceError(errFileAccessDenied), bucket, object)
|
||||
return ObjectInfo{}, toObjectErr(errors.Trace(errFileAccessDenied), bucket, object)
|
||||
}
|
||||
|
||||
// Validate input data size and it can never be less than zero.
|
||||
if data.Size() < 0 {
|
||||
return ObjectInfo{}, traceError(errInvalidArgument)
|
||||
return ObjectInfo{}, errors.Trace(errInvalidArgument)
|
||||
}
|
||||
|
||||
var wlk *lock.LockedFile
|
||||
@@ -604,7 +684,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
|
||||
fsMetaPath := pathJoin(bucketMetaDir, bucket, object, fsMetaJSONFile)
|
||||
wlk, err = fs.rwPool.Create(fsMetaPath)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, toObjectErr(traceError(err), bucket, object)
|
||||
return ObjectInfo{}, toObjectErr(errors.Trace(err), bucket, object)
|
||||
}
|
||||
// This close will allow for locks to be synchronized on `fs.json`.
|
||||
defer wlk.Close()
|
||||
@@ -633,7 +713,6 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
|
||||
bytesWritten, err := fsCreateFile(fsTmpObjPath, data, buf, data.Size())
|
||||
if err != nil {
|
||||
fsRemoveFile(fsTmpObjPath)
|
||||
errorIf(err, "Failed to create object %s/%s", bucket, object)
|
||||
return ObjectInfo{}, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
@@ -643,7 +722,7 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
|
||||
// bytes than specified in request header.
|
||||
if bytesWritten < data.Size() {
|
||||
fsRemoveFile(fsTmpObjPath)
|
||||
return ObjectInfo{}, traceError(IncompleteBody{})
|
||||
return ObjectInfo{}, errors.Trace(IncompleteBody{})
|
||||
}
|
||||
|
||||
// Delete the temporary object in the case of a
|
||||
@@ -676,8 +755,15 @@ func (fs fsObjects) PutObject(bucket string, object string, data *hash.Reader, m
|
||||
|
||||
// DeleteObject - deletes an object from a bucket, this operation is destructive
|
||||
// and there are no rollbacks supported.
|
||||
func (fs fsObjects) DeleteObject(bucket, object string) error {
|
||||
if err := checkBucketAndObjectNamesFS(bucket, object); err != nil {
|
||||
func (fs *fsObjects) DeleteObject(bucket, object string) error {
|
||||
// Acquire a write lock before deleting the object.
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, object)
|
||||
if err := objectLock.GetLock(globalOperationTimeout); err != nil {
|
||||
return err
|
||||
}
|
||||
defer objectLock.Unlock()
|
||||
|
||||
if err := checkDelObjArgs(bucket, object); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -694,7 +780,7 @@ func (fs fsObjects) DeleteObject(bucket, object string) error {
|
||||
defer rwlk.Close()
|
||||
}
|
||||
if lerr != nil && lerr != errFileNotFound {
|
||||
return toObjectErr(traceError(lerr), bucket, object)
|
||||
return toObjectErr(errors.Trace(lerr), bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -706,23 +792,17 @@ func (fs fsObjects) DeleteObject(bucket, object string) error {
|
||||
if bucket != minioMetaBucket {
|
||||
// Delete the metadata object.
|
||||
err := fsDeleteFile(minioMetaBucketDir, fsMetaPath)
|
||||
if err != nil && errorCause(err) != errFileNotFound {
|
||||
if err != nil && errors.Cause(err) != errFileNotFound {
|
||||
return toObjectErr(err, bucket, object)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// list of all errors that can be ignored in tree walk operation in FS
|
||||
var fsTreeWalkIgnoredErrs = append(baseIgnoredErrs, []error{
|
||||
errFileNotFound,
|
||||
errVolumeNotFound,
|
||||
}...)
|
||||
|
||||
// Returns function "listDir" of the type listDirFunc.
|
||||
// isLeaf - is used by listDir function to check if an entry
|
||||
// is a leaf or non-leaf entry.
|
||||
func (fs fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
|
||||
func (fs *fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
|
||||
// listDir - lists all the entries at a given prefix and given entry in the prefix.
|
||||
listDir := func(bucket, prefixDir, prefixEntry string) (entries []string, delayIsLeaf bool, err error) {
|
||||
entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
|
||||
@@ -739,7 +819,7 @@ func (fs fsObjects) listDirFactory(isLeaf isLeafFunc) listDirFunc {
|
||||
|
||||
// getObjectETag is a helper function, which returns only the md5sum
|
||||
// of the file on the disk.
|
||||
func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
func (fs *fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, entry, fsMetaJSONFile)
|
||||
|
||||
// Read `fs.json` to perhaps contend with
|
||||
@@ -747,7 +827,7 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
rlk, err := fs.rwPool.Open(fsMetaPath)
|
||||
// Ignore if `fs.json` is not available, this is true for pre-existing data.
|
||||
if err != nil && err != errFileNotFound {
|
||||
return "", toObjectErr(traceError(err), bucket, entry)
|
||||
return "", toObjectErr(errors.Trace(err), bucket, entry)
|
||||
}
|
||||
|
||||
// If file is not found, we don't need to proceed forward.
|
||||
@@ -761,7 +841,7 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
// Fetch the size of the underlying file.
|
||||
fi, err := rlk.LockedFile.Stat()
|
||||
if err != nil {
|
||||
return "", toObjectErr(traceError(err), bucket, entry)
|
||||
return "", toObjectErr(errors.Trace(err), bucket, entry)
|
||||
}
|
||||
|
||||
// `fs.json` can be empty due to previously failed
|
||||
@@ -775,12 +855,12 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
// make sure the underlying offsets don't move.
|
||||
fsMetaBuf, err := ioutil.ReadAll(io.NewSectionReader(rlk.LockedFile, 0, fi.Size()))
|
||||
if err != nil {
|
||||
return "", traceError(err)
|
||||
return "", errors.Trace(err)
|
||||
}
|
||||
|
||||
// Check if FS metadata is valid, if not return error.
|
||||
if !isFSMetaValid(parseFSVersion(fsMetaBuf), parseFSFormat(fsMetaBuf)) {
|
||||
return "", toObjectErr(traceError(errCorruptedFormat), bucket, entry)
|
||||
return "", toObjectErr(errors.Trace(errCorruptedFormat), bucket, entry)
|
||||
}
|
||||
|
||||
return extractETag(parseFSMetaMap(fsMetaBuf)), nil
|
||||
@@ -788,7 +868,7 @@ func (fs fsObjects) getObjectETag(bucket, entry string) (string, error) {
|
||||
|
||||
// ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
|
||||
// state for future re-entrant list requests.
|
||||
func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
func (fs *fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
if err := checkListObjsArgs(bucket, prefix, marker, delimiter, fs); err != nil {
|
||||
return loi, err
|
||||
}
|
||||
@@ -825,7 +905,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
|
||||
// Convert entry to ObjectInfo
|
||||
entryToObjectInfo := func(entry string) (objInfo ObjectInfo, err error) {
|
||||
// Protect the entry from concurrent deletes, or renames.
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, entry)
|
||||
objectLock := fs.nsMutex.NewNSLock(bucket, entry)
|
||||
if err = objectLock.GetRLock(globalListingTimeout); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
@@ -902,7 +982,7 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
|
||||
// For any walk error return right away.
|
||||
if walkResult.err != nil {
|
||||
// File not found is a valid case.
|
||||
if errorCause(walkResult.err) == errFileNotFound {
|
||||
if errors.Cause(walkResult.err) == errFileNotFound {
|
||||
return loi, nil
|
||||
}
|
||||
return loi, toObjectErr(walkResult.err, bucket, prefix)
|
||||
@@ -942,26 +1022,23 @@ func (fs fsObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKey
|
||||
}
|
||||
|
||||
// HealObject - no-op for fs. Valid only for XL.
|
||||
func (fs fsObjects) HealObject(bucket, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
func (fs *fsObjects) HealObject(bucket, object string, dryRun bool) (
|
||||
res madmin.HealResultItem, err error) {
|
||||
return res, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealBucket - no-op for fs, Valid only for XL.
|
||||
func (fs fsObjects) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
func (fs *fsObjects) HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem,
|
||||
error) {
|
||||
return nil, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - list all objects to be healed. Valid only for XL
|
||||
func (fs fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
func (fs *fsObjects) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - list all buckets to be healed. Valid only for XL
|
||||
func (fs fsObjects) ListBucketsHeal() ([]BucketInfo, error) {
|
||||
return []BucketInfo{}, traceError(NotImplemented{})
|
||||
}
|
||||
|
||||
func (fs fsObjects) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
|
||||
delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
return lmi, traceError(NotImplemented{})
|
||||
func (fs *fsObjects) ListBucketsHeal() ([]BucketInfo, error) {
|
||||
return []BucketInfo{}, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
+21
-19
@@ -22,6 +22,8 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// Tests for if parent directory is object
|
||||
@@ -178,7 +180,7 @@ func TestFSGetBucketInfo(t *testing.T) {
|
||||
|
||||
// Test with inexistant bucket
|
||||
_, err = fs.GetBucketInfo("a")
|
||||
if !isSameType(errorCause(err), BucketNameInvalid{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNameInvalid{}) {
|
||||
t.Fatal("BucketNameInvalid error not returned")
|
||||
}
|
||||
|
||||
@@ -186,7 +188,7 @@ func TestFSGetBucketInfo(t *testing.T) {
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
|
||||
_, err = fs.GetBucketInfo(bucketName)
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("BucketNotFound error not returned")
|
||||
}
|
||||
}
|
||||
@@ -209,7 +211,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
if _, ok := errorCause(err).(BucketNotFound); !ok {
|
||||
if _, ok := errors.Cause(err).(BucketNotFound); !ok {
|
||||
t.Fatalf("Expected error type BucketNotFound, got %#v", err)
|
||||
}
|
||||
|
||||
@@ -218,7 +220,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, bucket doesn't exist")
|
||||
}
|
||||
if _, ok := errorCause(err).(BucketNotFound); !ok {
|
||||
if _, ok := errors.Cause(err).(BucketNotFound); !ok {
|
||||
t.Fatalf("Expected error type BucketNotFound, got %#v", err)
|
||||
}
|
||||
|
||||
@@ -230,7 +232,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backend corruption occurred")
|
||||
}
|
||||
if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
|
||||
if nerr, ok := errors.Cause(err).(PrefixAccessDenied); !ok {
|
||||
t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
|
||||
} else {
|
||||
if nerr.Bucket != "bucket" {
|
||||
@@ -245,7 +247,7 @@ func TestFSPutObject(t *testing.T) {
|
||||
if err == nil {
|
||||
t.Fatal("Unexpected should fail here, backned corruption occurred")
|
||||
}
|
||||
if nerr, ok := errorCause(err).(PrefixAccessDenied); !ok {
|
||||
if nerr, ok := errors.Cause(err).(PrefixAccessDenied); !ok {
|
||||
t.Fatalf("Expected PrefixAccessDenied, got %#v", err)
|
||||
} else {
|
||||
if nerr.Bucket != "bucket" {
|
||||
@@ -272,19 +274,19 @@ func TestFSDeleteObject(t *testing.T) {
|
||||
obj.PutObject(bucketName, objectName, mustGetHashReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), nil)
|
||||
|
||||
// Test with invalid bucket name
|
||||
if err := fs.DeleteObject("fo", objectName); !isSameType(errorCause(err), BucketNameInvalid{}) {
|
||||
if err := fs.DeleteObject("fo", objectName); !isSameType(errors.Cause(err), BucketNameInvalid{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Test with bucket does not exist
|
||||
if err := fs.DeleteObject("foobucket", "fooobject"); !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if err := fs.DeleteObject("foobucket", "fooobject"); !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Test with invalid object name
|
||||
if err := fs.DeleteObject(bucketName, "\\"); !isSameType(errorCause(err), ObjectNameInvalid{}) {
|
||||
if err := fs.DeleteObject(bucketName, "\\"); !isSameType(errors.Cause(err), ObjectNameInvalid{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Test with object does not exist.
|
||||
if err := fs.DeleteObject(bucketName, "foooobject"); !isSameType(errorCause(err), ObjectNotFound{}) {
|
||||
if err := fs.DeleteObject(bucketName, "foooobject"); !isSameType(errors.Cause(err), ObjectNotFound{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Test with valid condition
|
||||
@@ -295,7 +297,7 @@ func TestFSDeleteObject(t *testing.T) {
|
||||
// Delete object should err disk not found.
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
if err := fs.DeleteObject(bucketName, objectName); err != nil {
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
}
|
||||
@@ -318,11 +320,11 @@ func TestFSDeleteBucket(t *testing.T) {
|
||||
}
|
||||
|
||||
// Test with an invalid bucket name
|
||||
if err = fs.DeleteBucket("fo"); !isSameType(errorCause(err), BucketNameInvalid{}) {
|
||||
if err = fs.DeleteBucket("fo"); !isSameType(errors.Cause(err), BucketNameInvalid{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Test with an inexistant bucket
|
||||
if err = fs.DeleteBucket("foobucket"); !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if err = fs.DeleteBucket("foobucket"); !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Test with a valid case
|
||||
@@ -335,7 +337,7 @@ func TestFSDeleteBucket(t *testing.T) {
|
||||
// Delete bucket should get error disk not found.
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
if err = fs.DeleteBucket(bucketName); err != nil {
|
||||
if !isSameType(errorCause(err), BucketNotFound{}) {
|
||||
if !isSameType(errors.Cause(err), BucketNotFound{}) {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
}
|
||||
@@ -378,7 +380,7 @@ func TestFSListBuckets(t *testing.T) {
|
||||
fs.fsPath = filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
|
||||
|
||||
if _, err := fs.ListBuckets(); err != nil {
|
||||
if errorCause(err) != errDiskNotFound {
|
||||
if errors.Cause(err) != errDiskNotFound {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
}
|
||||
@@ -386,7 +388,7 @@ func TestFSListBuckets(t *testing.T) {
|
||||
longPath := fmt.Sprintf("%0256d", 1)
|
||||
fs.fsPath = longPath
|
||||
if _, err := fs.ListBuckets(); err != nil {
|
||||
if errorCause(err) != errFileNameTooLong {
|
||||
if errors.Cause(err) != errFileNameTooLong {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
}
|
||||
@@ -398,8 +400,8 @@ func TestFSHealObject(t *testing.T) {
|
||||
defer os.RemoveAll(disk)
|
||||
|
||||
obj := initFSObjects(disk, t)
|
||||
_, _, err := obj.HealObject("bucket", "object")
|
||||
if err == nil || !isSameType(errorCause(err), NotImplemented{}) {
|
||||
_, err := obj.HealObject("bucket", "object", false)
|
||||
if err == nil || !isSameType(errors.Cause(err), NotImplemented{}) {
|
||||
t.Fatalf("Heal Object should return NotImplemented error ")
|
||||
}
|
||||
}
|
||||
@@ -411,7 +413,7 @@ func TestFSListObjectsHeal(t *testing.T) {
|
||||
|
||||
obj := initFSObjects(disk, t)
|
||||
_, err := obj.ListObjectsHeal("bucket", "prefix", "marker", "delimiter", 1000)
|
||||
if err == nil || !isSameType(errorCause(err), NotImplemented{}) {
|
||||
if err == nil || !isSameType(errors.Cause(err), NotImplemented{}) {
|
||||
t.Fatalf("Heal Object should return NotImplemented error ")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,74 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
func anonErrToObjectErr(statusCode int, params ...string) error {
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
|
||||
switch statusCode {
|
||||
case http.StatusNotFound:
|
||||
if object != "" {
|
||||
return ObjectNotFound{bucket, object}
|
||||
}
|
||||
return BucketNotFound{Bucket: bucket}
|
||||
case http.StatusBadRequest:
|
||||
if object != "" {
|
||||
return ObjectNameInvalid{bucket, object}
|
||||
}
|
||||
return BucketNameInvalid{Bucket: bucket}
|
||||
case http.StatusForbidden:
|
||||
fallthrough
|
||||
case http.StatusUnauthorized:
|
||||
return AllAccessDisabled{bucket, object}
|
||||
}
|
||||
|
||||
return errUnexpected
|
||||
}
|
||||
|
||||
// newCustomHTTPTransport returns a new http configuration
|
||||
// used while communicating with the cloud backends.
|
||||
// This sets the value for MaxIdleConns from 2 (go default) to
|
||||
// 100.
|
||||
func newCustomHTTPTransport() http.RoundTripper {
|
||||
return &http.Transport{
|
||||
Proxy: http.ProxyFromEnvironment,
|
||||
DialContext: (&net.Dialer{
|
||||
Timeout: 30 * time.Second,
|
||||
KeepAlive: 30 * time.Second,
|
||||
}).DialContext,
|
||||
MaxIdleConns: 100,
|
||||
IdleConnTimeout: 90 * time.Second,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ExpectContinueTimeout: 1 * time.Second,
|
||||
TLSClientConfig: &tls.Config{RootCAs: globalRootCAs},
|
||||
DisableCompression: true,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
|
||||
minio "github.com/minio/minio-go"
|
||||
)
|
||||
|
||||
var (
|
||||
// CanonicalizeETag provides canonicalizeETag function alias.
|
||||
CanonicalizeETag = canonicalizeETag
|
||||
|
||||
// MustGetUUID function alias.
|
||||
MustGetUUID = mustGetUUID
|
||||
|
||||
// ErrorIf provides errorIf function alias.
|
||||
ErrorIf = errorIf
|
||||
|
||||
// FatalIf provides fatalIf function alias.
|
||||
FatalIf = fatalIf
|
||||
)
|
||||
|
||||
// AnonErrToObjectErr - converts standard http codes into meaningful object layer errors.
|
||||
func AnonErrToObjectErr(statusCode int, params ...string) error {
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
|
||||
switch statusCode {
|
||||
case http.StatusNotFound:
|
||||
if object != "" {
|
||||
return ObjectNotFound{bucket, object}
|
||||
}
|
||||
return BucketNotFound{Bucket: bucket}
|
||||
case http.StatusBadRequest:
|
||||
if object != "" {
|
||||
return ObjectNameInvalid{bucket, object}
|
||||
}
|
||||
return BucketNameInvalid{Bucket: bucket}
|
||||
case http.StatusForbidden:
|
||||
fallthrough
|
||||
case http.StatusUnauthorized:
|
||||
return AllAccessDisabled{bucket, object}
|
||||
}
|
||||
|
||||
return errUnexpected
|
||||
}
|
||||
|
||||
// FromMinioClientMetadata converts minio metadata to map[string]string
|
||||
func FromMinioClientMetadata(metadata map[string][]string) map[string]string {
|
||||
mm := map[string]string{}
|
||||
for k, v := range metadata {
|
||||
mm[http.CanonicalHeaderKey(k)] = v[0]
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// FromMinioClientObjectPart converts minio ObjectPart to PartInfo
|
||||
func FromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
|
||||
return PartInfo{
|
||||
Size: op.Size,
|
||||
ETag: canonicalizeETag(op.ETag),
|
||||
LastModified: op.LastModified,
|
||||
PartNumber: op.PartNumber,
|
||||
}
|
||||
}
|
||||
|
||||
// FromMinioClientListPartsInfo converts minio ListObjectPartsResult to ListPartsInfo
|
||||
func FromMinioClientListPartsInfo(lopr minio.ListObjectPartsResult) ListPartsInfo {
|
||||
// Convert minio ObjectPart to PartInfo
|
||||
fromMinioClientObjectParts := func(parts []minio.ObjectPart) []PartInfo {
|
||||
toParts := make([]PartInfo, len(parts))
|
||||
for i, part := range parts {
|
||||
toParts[i] = FromMinioClientObjectPart(part)
|
||||
}
|
||||
return toParts
|
||||
}
|
||||
|
||||
return ListPartsInfo{
|
||||
UploadID: lopr.UploadID,
|
||||
Bucket: lopr.Bucket,
|
||||
Object: lopr.Key,
|
||||
StorageClass: "",
|
||||
PartNumberMarker: lopr.PartNumberMarker,
|
||||
NextPartNumberMarker: lopr.NextPartNumberMarker,
|
||||
MaxParts: lopr.MaxParts,
|
||||
IsTruncated: lopr.IsTruncated,
|
||||
EncodingType: lopr.EncodingType,
|
||||
Parts: fromMinioClientObjectParts(lopr.ObjectParts),
|
||||
}
|
||||
}
|
||||
|
||||
// FromMinioClientListMultipartsInfo converts minio ListMultipartUploadsResult to ListMultipartsInfo
|
||||
func FromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) ListMultipartsInfo {
|
||||
uploads := make([]MultipartInfo, len(lmur.Uploads))
|
||||
|
||||
for i, um := range lmur.Uploads {
|
||||
uploads[i] = MultipartInfo{
|
||||
Object: um.Key,
|
||||
UploadID: um.UploadID,
|
||||
Initiated: um.Initiated,
|
||||
}
|
||||
}
|
||||
|
||||
commonPrefixes := make([]string, len(lmur.CommonPrefixes))
|
||||
for i, cp := range lmur.CommonPrefixes {
|
||||
commonPrefixes[i] = cp.Prefix
|
||||
}
|
||||
|
||||
return ListMultipartsInfo{
|
||||
KeyMarker: lmur.KeyMarker,
|
||||
UploadIDMarker: lmur.UploadIDMarker,
|
||||
NextKeyMarker: lmur.NextKeyMarker,
|
||||
NextUploadIDMarker: lmur.NextUploadIDMarker,
|
||||
MaxUploads: int(lmur.MaxUploads),
|
||||
IsTruncated: lmur.IsTruncated,
|
||||
Uploads: uploads,
|
||||
Prefix: lmur.Prefix,
|
||||
Delimiter: lmur.Delimiter,
|
||||
CommonPrefixes: commonPrefixes,
|
||||
EncodingType: lmur.EncodingType,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// FromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
|
||||
func FromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
|
||||
userDefined := FromMinioClientMetadata(oi.Metadata)
|
||||
userDefined["Content-Type"] = oi.ContentType
|
||||
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: oi.Key,
|
||||
ModTime: oi.LastModified,
|
||||
Size: oi.Size,
|
||||
ETag: canonicalizeETag(oi.ETag),
|
||||
UserDefined: userDefined,
|
||||
ContentType: oi.ContentType,
|
||||
ContentEncoding: oi.Metadata.Get("Content-Encoding"),
|
||||
}
|
||||
}
|
||||
|
||||
// FromMinioClientListBucketV2Result converts minio ListBucketResult to ListObjectsInfo
|
||||
func FromMinioClientListBucketV2Result(bucket string, result minio.ListBucketV2Result) ListObjectsV2Info {
|
||||
objects := make([]ObjectInfo, len(result.Contents))
|
||||
|
||||
for i, oi := range result.Contents {
|
||||
objects[i] = FromMinioClientObjectInfo(bucket, oi)
|
||||
}
|
||||
|
||||
prefixes := make([]string, len(result.CommonPrefixes))
|
||||
for i, p := range result.CommonPrefixes {
|
||||
prefixes[i] = p.Prefix
|
||||
}
|
||||
|
||||
return ListObjectsV2Info{
|
||||
IsTruncated: result.IsTruncated,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
|
||||
ContinuationToken: result.ContinuationToken,
|
||||
NextContinuationToken: result.NextContinuationToken,
|
||||
}
|
||||
}
|
||||
|
||||
// FromMinioClientListBucketResult converts minio ListBucketResult to ListObjectsInfo
|
||||
func FromMinioClientListBucketResult(bucket string, result minio.ListBucketResult) ListObjectsInfo {
|
||||
objects := make([]ObjectInfo, len(result.Contents))
|
||||
|
||||
for i, oi := range result.Contents {
|
||||
objects[i] = FromMinioClientObjectInfo(bucket, oi)
|
||||
}
|
||||
|
||||
prefixes := make([]string, len(result.CommonPrefixes))
|
||||
for i, p := range result.CommonPrefixes {
|
||||
prefixes[i] = p.Prefix
|
||||
}
|
||||
|
||||
return ListObjectsInfo{
|
||||
IsTruncated: result.IsTruncated,
|
||||
NextMarker: result.NextMarker,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
}
|
||||
}
|
||||
|
||||
// FromMinioClientListBucketResultToV2Info converts minio ListBucketResult to ListObjectsV2Info
|
||||
func FromMinioClientListBucketResultToV2Info(bucket string, result minio.ListBucketResult) ListObjectsV2Info {
|
||||
objects := make([]ObjectInfo, len(result.Contents))
|
||||
|
||||
for i, oi := range result.Contents {
|
||||
objects[i] = FromMinioClientObjectInfo(bucket, oi)
|
||||
}
|
||||
|
||||
prefixes := make([]string, len(result.CommonPrefixes))
|
||||
for i, p := range result.CommonPrefixes {
|
||||
prefixes[i] = p.Prefix
|
||||
}
|
||||
|
||||
return ListObjectsV2Info{
|
||||
IsTruncated: result.IsTruncated,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
ContinuationToken: result.Marker,
|
||||
NextContinuationToken: result.NextMarker,
|
||||
}
|
||||
}
|
||||
|
||||
// ToMinioClientMetadata converts metadata to map[string][]string
|
||||
func ToMinioClientMetadata(metadata map[string]string) map[string]string {
|
||||
mm := make(map[string]string)
|
||||
for k, v := range metadata {
|
||||
mm[http.CanonicalHeaderKey(k)] = v
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// ToMinioClientCompletePart converts CompletePart to minio CompletePart
|
||||
func ToMinioClientCompletePart(part CompletePart) minio.CompletePart {
|
||||
return minio.CompletePart{
|
||||
ETag: part.ETag,
|
||||
PartNumber: part.PartNumber,
|
||||
}
|
||||
}
|
||||
|
||||
// ToMinioClientCompleteParts converts []CompletePart to minio []CompletePart
|
||||
func ToMinioClientCompleteParts(parts []CompletePart) []minio.CompletePart {
|
||||
mparts := make([]minio.CompletePart, len(parts))
|
||||
for i, part := range parts {
|
||||
mparts[i] = ToMinioClientCompletePart(part)
|
||||
}
|
||||
return mparts
|
||||
}
|
||||
|
||||
// ErrorRespToObjectError converts Minio errors to minio object layer errors.
|
||||
func ErrorRespToObjectError(err error, params ...string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
e, ok := err.(*errors.Error)
|
||||
if !ok {
|
||||
// Code should be fixed if this function is called without doing traceError()
|
||||
// Else handling different situations in this function makes this function complicated.
|
||||
errorIf(err, "Expected type *Error")
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.Cause
|
||||
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
|
||||
minioErr, ok := err.(minio.ErrorResponse)
|
||||
if !ok {
|
||||
// We don't interpret non Minio errors. As minio errors will
|
||||
// have StatusCode to help to convert to object errors.
|
||||
return e
|
||||
}
|
||||
|
||||
switch minioErr.Code {
|
||||
case "BucketAlreadyOwnedByYou":
|
||||
err = BucketAlreadyOwnedByYou{}
|
||||
case "BucketNotEmpty":
|
||||
err = BucketNotEmpty{}
|
||||
case "NoSuchBucketPolicy":
|
||||
err = PolicyNotFound{}
|
||||
case "InvalidBucketName":
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
case "NoSuchBucket":
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
case "NoSuchKey":
|
||||
if object != "" {
|
||||
err = ObjectNotFound{Bucket: bucket, Object: object}
|
||||
} else {
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
}
|
||||
case "XMinioInvalidObjectName":
|
||||
err = ObjectNameInvalid{}
|
||||
case "AccessDenied":
|
||||
err = PrefixAccessDenied{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
case "XAmzContentSHA256Mismatch":
|
||||
err = hash.SHA256Mismatch{}
|
||||
case "NoSuchUpload":
|
||||
err = InvalidUploadID{}
|
||||
case "EntityTooSmall":
|
||||
err = PartTooSmall{}
|
||||
}
|
||||
|
||||
e.Cause = err
|
||||
return e
|
||||
}
|
||||
+22
-38
@@ -61,7 +61,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
@@ -81,7 +81,6 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
objInfo, err := getObjectInfo(bucket, object)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to fetch object info.")
|
||||
apiErr := toAPIErrorCode(err)
|
||||
if apiErr == ErrNoSuchKey {
|
||||
apiErr = errAllowableObjectNotFound(bucket, r)
|
||||
@@ -129,7 +128,7 @@ func (api gatewayAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
setHeadGetRespHeaders(w, r.URL.Query())
|
||||
httpWriter := ioutil.WriteOnClose(w)
|
||||
// Reads the object at startOffset and writes to mw.
|
||||
if err = getObject(bucket, object, startOffset, length, httpWriter); err != nil {
|
||||
if err = getObject(bucket, object, startOffset, length, httpWriter, objInfo.ETag); err != nil {
|
||||
errorIf(err, "Unable to write to client.")
|
||||
if !httpWriter.HasWritten() {
|
||||
// Error response only if no data has been written to client yet. i.e if
|
||||
@@ -186,12 +185,19 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
bucket = vars["bucket"]
|
||||
object = vars["object"]
|
||||
|
||||
// Validate storage class metadata if present
|
||||
if _, ok := r.Header[amzStorageClassCanonical]; ok {
|
||||
if !isValidStorageClassMeta(r.Header.Get(amzStorageClassCanonical)) {
|
||||
writeErrorResponse(w, ErrInvalidStorageClass, r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: we should validate the object name here
|
||||
|
||||
// Get Content-Md5 sent by client and verify if valid
|
||||
md5Bytes, err := checkValidMD5(r.Header.Get("Content-Md5"))
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to validate content-md5 format.")
|
||||
writeErrorResponse(w, ErrInvalidDigest, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -203,7 +209,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
sizeStr := r.Header.Get("x-amz-decoded-content-length")
|
||||
size, err = strconv.ParseInt(sizeStr, 10, 64)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to parse `x-amz-decoded-content-length` into its integer value", sizeStr)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -222,7 +227,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
// Extract metadata to be saved from incoming HTTP header.
|
||||
metadata, err := extractMetadataFromHeader(r.Header)
|
||||
if err != nil {
|
||||
errorIf(err, "found invalid http request header")
|
||||
writeErrorResponse(w, ErrInternalError, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -246,14 +250,6 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
}
|
||||
|
||||
// Lock the object.
|
||||
objectLock := globalNSMutex.NewNSLock(bucket, object)
|
||||
if objectLock.GetLock(globalOperationTimeout) != nil {
|
||||
writeErrorResponse(w, ErrOperationTimedOut, r.URL)
|
||||
return
|
||||
}
|
||||
defer objectLock.Unlock()
|
||||
|
||||
var (
|
||||
// Make sure we hex encode md5sum here.
|
||||
md5hex = hex.EncodeToString(md5Bytes)
|
||||
@@ -286,7 +282,7 @@ func (api gatewayAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
case authTypePresigned, authTypeSigned:
|
||||
if s3Error := reqSignatureV4Verify(r, serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := reqSignatureV4Verify(r, globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
@@ -357,7 +353,7 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
@@ -377,7 +373,6 @@ func (api gatewayAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
objInfo, err := getObjectInfo(bucket, object)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to fetch object info.")
|
||||
apiErr := toAPIErrorCode(err)
|
||||
if apiErr == ErrNoSuchKey {
|
||||
apiErr = errAllowableObjectNotFound(bucket, r)
|
||||
@@ -429,7 +424,7 @@ func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -440,7 +435,6 @@ func (api gatewayAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *h
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -492,7 +486,7 @@ func (api gatewayAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -503,7 +497,6 @@ func (api gatewayAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -526,7 +519,7 @@ func (api gatewayAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -537,21 +530,18 @@ func (api gatewayAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *h
|
||||
// Before proceeding validate if bucket exists.
|
||||
_, err := objAPI.GetBucketInfo(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to find bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
bp, err := objAPI.GetBucketPolicies(bucket)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
policyBytes, err := json.Marshal(bp)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to read bucket policy.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -596,7 +586,7 @@ func (api gatewayAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
// PutBucket does not have any bucket action.
|
||||
s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion())
|
||||
s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
@@ -623,7 +613,6 @@ func (api gatewayAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Re
|
||||
// Proceed to creating a bucket.
|
||||
err := objectAPI.MakeBucketWithLocation(bucket, location)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to create a bucket.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -643,7 +632,7 @@ func (api gatewayAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http
|
||||
}
|
||||
|
||||
// DeleteBucket does not have any bucket action.
|
||||
if s3Error := checkRequestAuthType(r, "", "", serverConfig.GetRegion()); s3Error != ErrNone {
|
||||
if s3Error := checkRequestAuthType(r, "", "", globalServerConfig.GetRegion()); s3Error != ErrNone {
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
return
|
||||
}
|
||||
@@ -653,7 +642,6 @@ func (api gatewayAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http
|
||||
|
||||
// Attempt to delete bucket.
|
||||
if err := objectAPI.DeleteBucket(bucket); err != nil {
|
||||
errorIf(err, "Unable to delete a bucket.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -690,7 +678,7 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
@@ -725,7 +713,6 @@ func (api gatewayAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *htt
|
||||
// marshalled into S3 compatible XML header.
|
||||
listObjectsInfo, err := listObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list objects.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -764,7 +751,7 @@ func (api gatewayAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *htt
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
@@ -805,7 +792,6 @@ func (api gatewayAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *htt
|
||||
// serialized as XML and sent as S3 compatible response body.
|
||||
listObjectsV2Info, err := listObjectsV2(bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
|
||||
if err != nil {
|
||||
errorIf(err, "Unable to list objects. Args to listObjectsV2 are bucket=%s, prefix=%s, token=%s, delimiter=%s", bucket, prefix, token, delimiter)
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -843,7 +829,7 @@ func (api gatewayAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.R
|
||||
return
|
||||
}
|
||||
case authTypeSigned, authTypePresigned:
|
||||
s3Error := isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
s3Error := isReqAuthenticated(r, globalServerConfig.GetRegion())
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
writeErrorResponse(w, s3Error, r.URL)
|
||||
@@ -863,7 +849,6 @@ func (api gatewayAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
if _, err := getBucketInfo(bucket); err != nil {
|
||||
errorIf(err, "Unable to fetch bucket info.")
|
||||
writeErrorResponseHeadersOnly(w, toAPIErrorCode(err))
|
||||
return
|
||||
}
|
||||
@@ -898,7 +883,7 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
|
||||
s3Error := isReqAuthenticated(r, globalMinioDefaultRegion)
|
||||
if s3Error == ErrInvalidRegion {
|
||||
// Clients like boto3 send getBucketLocation() call signed with region that is configured.
|
||||
s3Error = isReqAuthenticated(r, serverConfig.GetRegion())
|
||||
s3Error = isReqAuthenticated(r, globalServerConfig.GetRegion())
|
||||
}
|
||||
if s3Error != ErrNone {
|
||||
errorIf(errSignatureMismatch, "%s", dumpRequest(r))
|
||||
@@ -919,7 +904,6 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
if _, err := getBucketInfo(bucket); err != nil {
|
||||
errorIf(err, "Unable to fetch bucket info.")
|
||||
writeErrorResponse(w, toAPIErrorCode(err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -927,7 +911,7 @@ func (api gatewayAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r
|
||||
// Generate response.
|
||||
encodedSuccessResponse := encodeResponse(LocationResponse{})
|
||||
// Get current region.
|
||||
region := serverConfig.GetRegion()
|
||||
region := globalServerConfig.GetRegion()
|
||||
if region != globalMinioDefaultRegion {
|
||||
encodedSuccessResponse = encodeResponse(LocationResponse{
|
||||
Location: region,
|
||||
|
||||
+37
-41
@@ -17,7 +17,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"os"
|
||||
@@ -28,6 +27,7 @@ import (
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
miniohttp "github.com/minio/minio/pkg/http"
|
||||
)
|
||||
|
||||
@@ -40,36 +40,15 @@ var (
|
||||
}
|
||||
)
|
||||
|
||||
// Gateway represents a gateway backend.
|
||||
type Gateway interface {
|
||||
// Name returns the unique name of the gateway.
|
||||
Name() string
|
||||
// NewGatewayLayer returns a new gateway layer.
|
||||
NewGatewayLayer() (GatewayLayer, error)
|
||||
}
|
||||
|
||||
// RegisterGatewayCommand registers a new command for gateway.
|
||||
func RegisterGatewayCommand(cmd cli.Command) error {
|
||||
// We should not have multiple subcommands with same name.
|
||||
for _, c := range gatewayCmd.Subcommands {
|
||||
if c.Name == cmd.Name {
|
||||
return fmt.Errorf("duplicate gateway: %s", cmd.Name)
|
||||
}
|
||||
}
|
||||
|
||||
cmd.Flags = append(append(cmd.Flags, append(cmd.Flags, serverFlags...)...), globalFlags...)
|
||||
gatewayCmd.Subcommands = append(gatewayCmd.Subcommands, cmd)
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustRegisterGatewayCommand is like RegisterGatewayCommand but panics instead of returning error.
|
||||
func MustRegisterGatewayCommand(cmd cli.Command) {
|
||||
if err := RegisterGatewayCommand(cmd); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Return endpoint.
|
||||
func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) {
|
||||
// ParseGatewayEndpoint - Return endpoint.
|
||||
func ParseGatewayEndpoint(arg string) (endPoint string, secure bool, err error) {
|
||||
schemeSpecified := len(strings.Split(arg, "://")) > 1
|
||||
if !schemeSpecified {
|
||||
// Default connection will be "secure".
|
||||
@@ -91,8 +70,8 @@ func parseGatewayEndpoint(arg string) (endPoint string, secure bool, err error)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate gateway arguments.
|
||||
func validateGatewayArguments(serverAddr, endpointAddr string) error {
|
||||
// ValidateGatewayArguments - Validate gateway arguments.
|
||||
func ValidateGatewayArguments(serverAddr, endpointAddr string) error {
|
||||
if err := CheckLocalServerAddr(serverAddr); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -115,14 +94,31 @@ func validateGatewayArguments(serverAddr, endpointAddr string) error {
|
||||
return err
|
||||
}
|
||||
if sameTarget {
|
||||
return errors.New("endpoint points to the local gateway")
|
||||
return fmt.Errorf("endpoint points to the local gateway")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway <name>'.
|
||||
func startGateway(ctx *cli.Context, gw Gateway) {
|
||||
// StartGateway - handler for 'minio gateway <name>'.
|
||||
func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
if gw == nil {
|
||||
fatalIf(errUnexpected, "Gateway implementation not initialized, exiting.")
|
||||
}
|
||||
|
||||
// Validate if we have access, secret set through environment.
|
||||
gatewayName := gw.Name()
|
||||
if ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
// Get "json" flag from command line argument and
|
||||
// enable json and quite modes if jason flag is turned on.
|
||||
jsonFlag := ctx.Bool("json") || ctx.GlobalBool("json")
|
||||
if jsonFlag {
|
||||
log.EnableJSON()
|
||||
}
|
||||
|
||||
// Get quiet flag from command line argument.
|
||||
quietFlag := ctx.Bool("quiet") || ctx.GlobalBool("quiet")
|
||||
if quietFlag {
|
||||
@@ -142,9 +138,8 @@ func startGateway(ctx *cli.Context, gw Gateway) {
|
||||
handleCommonEnvVars()
|
||||
|
||||
// Validate if we have access, secret set through environment.
|
||||
gatewayName := gw.Name()
|
||||
if !globalIsEnvCreds {
|
||||
errorIf(errors.New("Access and secret keys not set"), "Access and Secret keys should be set through ENVs for backend [%s]", gatewayName)
|
||||
errorIf(fmt.Errorf("Access and secret keys not set"), "Access and Secret keys should be set through ENVs for backend [%s]", gatewayName)
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
@@ -154,24 +149,20 @@ func startGateway(ctx *cli.Context, gw Gateway) {
|
||||
// Initialize gateway config.
|
||||
initConfig()
|
||||
|
||||
// Enable loggers as per configuration file.
|
||||
enableLoggers()
|
||||
|
||||
// Init the error tracing module.
|
||||
initError()
|
||||
errors.Init(GOPATH, "github.com/minio/minio")
|
||||
|
||||
// Check and load SSL certificates.
|
||||
var err error
|
||||
globalPublicCerts, globalRootCAs, globalTLSCertificate, globalIsSSL, err = getSSLConfig()
|
||||
fatalIf(err, "Invalid SSL certificate file")
|
||||
|
||||
// Set system resources to maximum.
|
||||
errorIf(setMaxResources(), "Unable to change resource limit")
|
||||
|
||||
initNSLock(false) // Enable local namespace lock.
|
||||
|
||||
if ctx.Args().First() == "help" {
|
||||
cli.ShowCommandHelpAndExit(ctx, gatewayName, 1)
|
||||
}
|
||||
|
||||
newObject, err := gw.NewGatewayLayer()
|
||||
newObject, err := gw.NewGatewayLayer(globalServerConfig.GetCredential())
|
||||
fatalIf(err, "Unable to initialize gateway layer")
|
||||
|
||||
router := mux.NewRouter().SkipClean(true)
|
||||
@@ -230,6 +221,11 @@ func startGateway(ctx *cli.Context, gw Gateway) {
|
||||
// Check update mode.
|
||||
checkUpdate(mode)
|
||||
|
||||
// Print a warning message if gateway is not ready for production before the startup banner.
|
||||
if !gw.Production() {
|
||||
log.Println(colorYellow("\n *** Warning: Not Ready for Production ***"))
|
||||
}
|
||||
|
||||
// Print gateway startup message.
|
||||
printGatewayStartupMessage(getAPIEndpoints(gatewayAddr), gatewayName)
|
||||
}
|
||||
|
||||
@@ -32,16 +32,6 @@ func TestRegisterGatewayCommand(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
|
||||
}
|
||||
|
||||
// Should returns 'duplicated' error
|
||||
err = RegisterGatewayCommand(cmd)
|
||||
if err == nil {
|
||||
t.Errorf("RegisterGatewayCommand twice with same name should return error")
|
||||
} else {
|
||||
if err.Error() != "duplicate gateway: test" {
|
||||
t.Errorf("RegisterGatewayCommand got unexpected error: %s", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test parseGatewayEndpoint
|
||||
@@ -62,7 +52,7 @@ func TestParseGatewayEndpoint(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, test := range testCases {
|
||||
endPoint, secure, err := parseGatewayEndpoint(test.arg)
|
||||
endPoint, secure, err := ParseGatewayEndpoint(test.arg)
|
||||
errReturned := err != nil
|
||||
|
||||
if endPoint != test.endPoint ||
|
||||
@@ -97,7 +87,7 @@ func TestValidateGatewayArguments(t *testing.T) {
|
||||
{":9000", nonLoopBackIP + ":9000", false},
|
||||
}
|
||||
for i, test := range testCases {
|
||||
err := validateGatewayArguments(test.serverAddr, test.endpointAddr)
|
||||
err := ValidateGatewayArguments(test.serverAddr, test.endpointAddr)
|
||||
if test.valid && err != nil {
|
||||
t.Errorf("Test %d expected not to return error but got %s", i+1, err)
|
||||
}
|
||||
|
||||
+18
-2
@@ -22,14 +22,30 @@ import (
|
||||
|
||||
router "github.com/gorilla/mux"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
// GatewayLayer - Interface to implement gateway mode.
|
||||
// GatewayMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API.
|
||||
const GatewayMinioSysTmp = "minio.sys.tmp/"
|
||||
|
||||
// Gateway represents a gateway backend.
|
||||
type Gateway interface {
|
||||
// Name returns the unique name of the gateway.
|
||||
Name() string
|
||||
|
||||
// NewGatewayLayer returns a new gateway layer.
|
||||
NewGatewayLayer(creds auth.Credentials) (GatewayLayer, error)
|
||||
|
||||
// Returns true if gateway is ready for production.
|
||||
Production() bool
|
||||
}
|
||||
|
||||
// GatewayLayer - interface to implement gateway mode.
|
||||
type GatewayLayer interface {
|
||||
ObjectLayer
|
||||
|
||||
AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error)
|
||||
AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error)
|
||||
AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error)
|
||||
|
||||
AnonPutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (ObjectInfo, error)
|
||||
|
||||
@@ -1,639 +0,0 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2017 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http"
|
||||
|
||||
"github.com/minio/cli"
|
||||
minio "github.com/minio/minio-go"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
)
|
||||
|
||||
const (
|
||||
s3Backend = "s3"
|
||||
)
|
||||
|
||||
func init() {
|
||||
const s3GatewayTemplate = `NAME:
|
||||
{{.HelpName}} - {{.Usage}}
|
||||
|
||||
USAGE:
|
||||
{{.HelpName}} {{if .VisibleFlags}}[FLAGS]{{end}} [ENDPOINT]
|
||||
{{if .VisibleFlags}}
|
||||
FLAGS:
|
||||
{{range .VisibleFlags}}{{.}}
|
||||
{{end}}{{end}}
|
||||
ENDPOINT:
|
||||
S3 server endpoint. Default ENDPOINT is https://s3.amazonaws.com
|
||||
|
||||
ENVIRONMENT VARIABLES:
|
||||
ACCESS:
|
||||
MINIO_ACCESS_KEY: Username or access key of S3 storage.
|
||||
MINIO_SECRET_KEY: Password or secret key of S3 storage.
|
||||
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend.
|
||||
$ export MINIO_ACCESS_KEY=accesskey
|
||||
$ export MINIO_SECRET_KEY=secretkey
|
||||
$ {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for S3 backend on custom endpoint.
|
||||
$ export MINIO_ACCESS_KEY=Q3AM3UQ867SPQQA43P2F
|
||||
$ export MINIO_SECRET_KEY=zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG
|
||||
$ {{.HelpName}} https://play.minio.io:9000
|
||||
`
|
||||
|
||||
MustRegisterGatewayCommand(cli.Command{
|
||||
Name: s3Backend,
|
||||
Usage: "Amazon Simple Storage Service (S3).",
|
||||
Action: s3GatewayMain,
|
||||
CustomHelpTemplate: s3GatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
})
|
||||
}
|
||||
|
||||
// Handler for 'minio gateway s3' command line.
|
||||
func s3GatewayMain(ctx *cli.Context) {
|
||||
// Validate gateway arguments.
|
||||
host := ctx.Args().First()
|
||||
// Validate gateway arguments.
|
||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||
|
||||
startGateway(ctx, &S3Gateway{host})
|
||||
}
|
||||
|
||||
// S3Gateway implements Gateway.
|
||||
type S3Gateway struct {
|
||||
host string
|
||||
}
|
||||
|
||||
// Name implements Gateway interface.
|
||||
func (g *S3Gateway) Name() string {
|
||||
return s3Backend
|
||||
}
|
||||
|
||||
// NewGatewayLayer returns s3 gatewaylayer.
|
||||
func (g *S3Gateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||
return newS3GatewayLayer(g.host)
|
||||
}
|
||||
|
||||
// s3ToObjectError converts Minio errors to minio object layer errors.
|
||||
func s3ToObjectError(err error, params ...string) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
e, ok := err.(*Error)
|
||||
if !ok {
|
||||
// Code should be fixed if this function is called without doing traceError()
|
||||
// Else handling different situations in this function makes this function complicated.
|
||||
errorIf(err, "Expected type *Error")
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.e
|
||||
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
bucket = params[0]
|
||||
}
|
||||
if len(params) == 2 {
|
||||
object = params[1]
|
||||
}
|
||||
|
||||
minioErr, ok := err.(minio.ErrorResponse)
|
||||
if !ok {
|
||||
// We don't interpret non Minio errors. As minio errors will
|
||||
// have StatusCode to help to convert to object errors.
|
||||
return e
|
||||
}
|
||||
|
||||
switch minioErr.Code {
|
||||
case "BucketAlreadyOwnedByYou":
|
||||
err = BucketAlreadyOwnedByYou{}
|
||||
case "BucketNotEmpty":
|
||||
err = BucketNotEmpty{}
|
||||
case "NoSuchBucketPolicy":
|
||||
err = PolicyNotFound{}
|
||||
case "InvalidBucketName":
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
case "NoSuchBucket":
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
case "NoSuchKey":
|
||||
if object != "" {
|
||||
err = ObjectNotFound{Bucket: bucket, Object: object}
|
||||
} else {
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
}
|
||||
case "XMinioInvalidObjectName":
|
||||
err = ObjectNameInvalid{}
|
||||
case "AccessDenied":
|
||||
err = PrefixAccessDenied{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
case "XAmzContentSHA256Mismatch":
|
||||
err = hash.SHA256Mismatch{}
|
||||
case "NoSuchUpload":
|
||||
err = InvalidUploadID{}
|
||||
case "EntityTooSmall":
|
||||
err = PartTooSmall{}
|
||||
}
|
||||
|
||||
e.e = err
|
||||
return e
|
||||
}
|
||||
|
||||
// s3Objects implements gateway for Minio and S3 compatible object storage servers.
|
||||
type s3Objects struct {
|
||||
gatewayUnsupported
|
||||
Client *minio.Core
|
||||
anonClient *minio.Core
|
||||
}
|
||||
|
||||
// newS3GatewayLayer returns s3 gatewaylayer
|
||||
func newS3GatewayLayer(host string) (GatewayLayer, error) {
|
||||
var err error
|
||||
var endpoint string
|
||||
var secure = true
|
||||
|
||||
// Validate host parameters.
|
||||
if host != "" {
|
||||
// Override default params if the host is provided
|
||||
endpoint, secure, err = parseGatewayEndpoint(host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Default endpoint parameters
|
||||
if endpoint == "" {
|
||||
endpoint = "s3.amazonaws.com"
|
||||
}
|
||||
|
||||
creds := serverConfig.GetCredential()
|
||||
|
||||
// Initialize minio client object.
|
||||
client, err := minio.NewCore(endpoint, creds.AccessKey, creds.SecretKey, secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
anonClient, err := minio.NewCore(endpoint, "", "", secure)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
anonClient.SetCustomTransport(newCustomHTTPTransport())
|
||||
|
||||
return &s3Objects{
|
||||
Client: client,
|
||||
anonClient: anonClient,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Shutdown saves any gateway metadata to disk
|
||||
// if necessary and reload upon next restart.
|
||||
func (l *s3Objects) Shutdown() error {
|
||||
// TODO
|
||||
return nil
|
||||
}
|
||||
|
||||
// StorageInfo is not relevant to S3 backend.
|
||||
func (l *s3Objects) StorageInfo() (si StorageInfo) {
|
||||
return si
|
||||
}
|
||||
|
||||
// MakeBucket creates a new container on S3 backend.
|
||||
func (l *s3Objects) MakeBucketWithLocation(bucket, location string) error {
|
||||
err := l.Client.MakeBucket(bucket, location)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// GetBucketInfo gets bucket metadata..
|
||||
func (l *s3Objects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
// Verify if bucket name is valid.
|
||||
// We are using a separate helper function here to validate bucket
|
||||
// names instead of IsValidBucketName() because there is a possibility
|
||||
// that certains users might have buckets which are non-DNS compliant
|
||||
// in us-east-1 and we might severely restrict them by not allowing
|
||||
// access to these buckets.
|
||||
// Ref - http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html
|
||||
if s3utils.CheckValidBucketName(bucket) != nil {
|
||||
return bi, traceError(BucketNameInvalid{Bucket: bucket})
|
||||
}
|
||||
|
||||
buckets, err := l.Client.ListBuckets()
|
||||
if err != nil {
|
||||
return bi, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
for _, bi := range buckets {
|
||||
if bi.Name != bucket {
|
||||
continue
|
||||
}
|
||||
|
||||
return BucketInfo{
|
||||
Name: bi.Name,
|
||||
Created: bi.CreationDate,
|
||||
}, nil
|
||||
}
|
||||
|
||||
return bi, traceError(BucketNotFound{Bucket: bucket})
|
||||
}
|
||||
|
||||
// ListBuckets lists all S3 buckets
|
||||
func (l *s3Objects) ListBuckets() ([]BucketInfo, error) {
|
||||
buckets, err := l.Client.ListBuckets()
|
||||
if err != nil {
|
||||
return nil, s3ToObjectError(traceError(err))
|
||||
}
|
||||
|
||||
b := make([]BucketInfo, len(buckets))
|
||||
for i, bi := range buckets {
|
||||
b[i] = BucketInfo{
|
||||
Name: bi.Name,
|
||||
Created: bi.CreationDate,
|
||||
}
|
||||
}
|
||||
|
||||
return b, err
|
||||
}
|
||||
|
||||
// DeleteBucket deletes a bucket on S3
|
||||
func (l *s3Objects) DeleteBucket(bucket string) error {
|
||||
err := l.Client.RemoveBucket(bucket)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListObjects lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3Objects) ListObjects(bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
result, err := l.Client.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
return loi, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
return fromMinioClientListBucketResult(bucket, result), nil
|
||||
}
|
||||
|
||||
// ListObjectsV2 lists all blobs in S3 bucket filtered by prefix
|
||||
func (l *s3Objects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (loi ListObjectsV2Info, e error) {
|
||||
result, err := l.Client.ListObjectsV2(bucket, prefix, continuationToken, fetchOwner, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
return loi, s3ToObjectError(traceError(err), bucket)
|
||||
}
|
||||
|
||||
return fromMinioClientListBucketV2Result(bucket, result), nil
|
||||
}
|
||||
|
||||
// fromMinioClientListBucketV2Result converts minio ListBucketResult to ListObjectsInfo
|
||||
func fromMinioClientListBucketV2Result(bucket string, result minio.ListBucketV2Result) ListObjectsV2Info {
|
||||
objects := make([]ObjectInfo, len(result.Contents))
|
||||
|
||||
for i, oi := range result.Contents {
|
||||
objects[i] = fromMinioClientObjectInfo(bucket, oi)
|
||||
}
|
||||
|
||||
prefixes := make([]string, len(result.CommonPrefixes))
|
||||
for i, p := range result.CommonPrefixes {
|
||||
prefixes[i] = p.Prefix
|
||||
}
|
||||
|
||||
return ListObjectsV2Info{
|
||||
IsTruncated: result.IsTruncated,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
|
||||
ContinuationToken: result.ContinuationToken,
|
||||
NextContinuationToken: result.NextContinuationToken,
|
||||
}
|
||||
}
|
||||
|
||||
// fromMinioClientListBucketResult converts minio ListBucketResult to ListObjectsInfo
|
||||
func fromMinioClientListBucketResult(bucket string, result minio.ListBucketResult) ListObjectsInfo {
|
||||
objects := make([]ObjectInfo, len(result.Contents))
|
||||
|
||||
for i, oi := range result.Contents {
|
||||
objects[i] = fromMinioClientObjectInfo(bucket, oi)
|
||||
}
|
||||
|
||||
prefixes := make([]string, len(result.CommonPrefixes))
|
||||
for i, p := range result.CommonPrefixes {
|
||||
prefixes[i] = p.Prefix
|
||||
}
|
||||
|
||||
return ListObjectsInfo{
|
||||
IsTruncated: result.IsTruncated,
|
||||
NextMarker: result.NextMarker,
|
||||
Prefixes: prefixes,
|
||||
Objects: objects,
|
||||
}
|
||||
}
|
||||
|
||||
// GetObject reads an object from S3. Supports additional
|
||||
// parameters like offset and length which are synonymous with
|
||||
// HTTP Range requests.
|
||||
//
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (l *s3Objects) GetObject(bucket string, key string, startOffset int64, length int64, writer io.Writer) error {
|
||||
if length < 0 && length != -1 {
|
||||
return s3ToObjectError(traceError(errInvalidArgument), bucket, key)
|
||||
}
|
||||
|
||||
opts := minio.GetObjectOptions{}
|
||||
if startOffset >= 0 && length >= 0 {
|
||||
if err := opts.SetRange(startOffset, startOffset+length-1); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
}
|
||||
object, _, err := l.Client.GetObject(bucket, key, opts)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
defer object.Close()
|
||||
|
||||
if _, err := io.Copy(writer, object); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, key)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fromMinioClientObjectInfo converts minio ObjectInfo to gateway ObjectInfo
|
||||
func fromMinioClientObjectInfo(bucket string, oi minio.ObjectInfo) ObjectInfo {
|
||||
userDefined := fromMinioClientMetadata(oi.Metadata)
|
||||
userDefined["Content-Type"] = oi.ContentType
|
||||
|
||||
return ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: oi.Key,
|
||||
ModTime: oi.LastModified,
|
||||
Size: oi.Size,
|
||||
ETag: canonicalizeETag(oi.ETag),
|
||||
UserDefined: userDefined,
|
||||
ContentType: oi.ContentType,
|
||||
ContentEncoding: oi.Metadata.Get("Content-Encoding"),
|
||||
}
|
||||
}
|
||||
|
||||
// GetObjectInfo reads object info and replies back ObjectInfo
|
||||
func (l *s3Objects) GetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
|
||||
oi, err := l.Client.StatObject(bucket, object, minio.StatObjectOptions{})
|
||||
if err != nil {
|
||||
return ObjectInfo{}, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
|
||||
// PutObject creates a new object with the incoming data,
|
||||
func (l *s3Objects) PutObject(bucket string, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
oi, err := l.Client.PutObject(bucket, object, data, data.Size(), data.MD5(), data.SHA256(), toMinioClientMetadata(metadata))
|
||||
if err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectInfo(bucket, oi), nil
|
||||
}
|
||||
|
||||
// CopyObject copies an object from source bucket to a destination bucket.
|
||||
func (l *s3Objects) CopyObject(srcBucket string, srcObject string, dstBucket string, dstObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
// Set this header such that following CopyObject() always sets the right metadata on the destination.
|
||||
// metadata input is already a trickled down value from interpreting x-amz-metadata-directive at
|
||||
// handler layer. So what we have right now is supposed to be applied on the destination object anyways.
|
||||
// So preserve it by adding "REPLACE" directive to save all the metadata set by CopyObject API.
|
||||
metadata["x-amz-metadata-directive"] = "REPLACE"
|
||||
if _, err = l.Client.CopyObject(srcBucket, srcObject, dstBucket, dstObject, metadata); err != nil {
|
||||
return objInfo, s3ToObjectError(traceError(err), srcBucket, srcObject)
|
||||
}
|
||||
return l.GetObjectInfo(dstBucket, dstObject)
|
||||
}
|
||||
|
||||
// DeleteObject deletes a blob in bucket
|
||||
func (l *s3Objects) DeleteObject(bucket string, object string) error {
|
||||
err := l.Client.RemoveObject(bucket, object)
|
||||
if err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// fromMinioClientMultipartInfo converts ObjectMultipartInfo to MultipartInfo
|
||||
func fromMinioClientMultipartInfo(omi minio.ObjectMultipartInfo) MultipartInfo {
|
||||
return MultipartInfo{
|
||||
Object: omi.Key,
|
||||
UploadID: omi.UploadID,
|
||||
Initiated: omi.Initiated,
|
||||
}
|
||||
}
|
||||
|
||||
// fromMinioClientListMultipartsInfo converts minio ListMultipartUploadsResult to ListMultipartsInfo
|
||||
func fromMinioClientListMultipartsInfo(lmur minio.ListMultipartUploadsResult) ListMultipartsInfo {
|
||||
uploads := make([]MultipartInfo, len(lmur.Uploads))
|
||||
|
||||
for i, um := range lmur.Uploads {
|
||||
uploads[i] = fromMinioClientMultipartInfo(um)
|
||||
}
|
||||
|
||||
commonPrefixes := make([]string, len(lmur.CommonPrefixes))
|
||||
for i, cp := range lmur.CommonPrefixes {
|
||||
commonPrefixes[i] = cp.Prefix
|
||||
}
|
||||
|
||||
return ListMultipartsInfo{
|
||||
KeyMarker: lmur.KeyMarker,
|
||||
UploadIDMarker: lmur.UploadIDMarker,
|
||||
NextKeyMarker: lmur.NextKeyMarker,
|
||||
NextUploadIDMarker: lmur.NextUploadIDMarker,
|
||||
MaxUploads: int(lmur.MaxUploads),
|
||||
IsTruncated: lmur.IsTruncated,
|
||||
Uploads: uploads,
|
||||
Prefix: lmur.Prefix,
|
||||
Delimiter: lmur.Delimiter,
|
||||
CommonPrefixes: commonPrefixes,
|
||||
EncodingType: lmur.EncodingType,
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// ListMultipartUploads lists all multipart uploads.
|
||||
func (l *s3Objects) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
result, err := l.Client.ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
if err != nil {
|
||||
return lmi, err
|
||||
}
|
||||
|
||||
return fromMinioClientListMultipartsInfo(result), nil
|
||||
}
|
||||
|
||||
// fromMinioClientMetadata converts minio metadata to map[string]string
|
||||
func fromMinioClientMetadata(metadata map[string][]string) map[string]string {
|
||||
mm := map[string]string{}
|
||||
for k, v := range metadata {
|
||||
mm[http.CanonicalHeaderKey(k)] = v[0]
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// toMinioClientMetadata converts metadata to map[string][]string
|
||||
func toMinioClientMetadata(metadata map[string]string) map[string]string {
|
||||
mm := map[string]string{}
|
||||
for k, v := range metadata {
|
||||
mm[http.CanonicalHeaderKey(k)] = v
|
||||
}
|
||||
return mm
|
||||
}
|
||||
|
||||
// NewMultipartUpload upload object in multiple parts
|
||||
func (l *s3Objects) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
// Create PutObject options
|
||||
opts := minio.PutObjectOptions{UserMetadata: metadata}
|
||||
uploadID, err = l.Client.NewMultipartUpload(bucket, object, opts)
|
||||
if err != nil {
|
||||
return uploadID, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// fromMinioClientObjectPart converts minio ObjectPart to PartInfo
|
||||
func fromMinioClientObjectPart(op minio.ObjectPart) PartInfo {
|
||||
return PartInfo{
|
||||
Size: op.Size,
|
||||
ETag: canonicalizeETag(op.ETag),
|
||||
LastModified: op.LastModified,
|
||||
PartNumber: op.PartNumber,
|
||||
}
|
||||
}
|
||||
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (l *s3Objects) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, e error) {
|
||||
info, err := l.Client.PutObjectPart(bucket, object, uploadID, partID, data, data.Size(), data.MD5(), data.SHA256())
|
||||
if err != nil {
|
||||
return pi, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return fromMinioClientObjectPart(info), nil
|
||||
}
|
||||
|
||||
// fromMinioClientObjectParts converts minio ObjectPart to PartInfo
|
||||
func fromMinioClientObjectParts(parts []minio.ObjectPart) []PartInfo {
|
||||
toParts := make([]PartInfo, len(parts))
|
||||
for i, part := range parts {
|
||||
toParts[i] = fromMinioClientObjectPart(part)
|
||||
}
|
||||
return toParts
|
||||
}
|
||||
|
||||
// fromMinioClientListPartsInfo converts minio ListObjectPartsResult to ListPartsInfo
|
||||
func fromMinioClientListPartsInfo(lopr minio.ListObjectPartsResult) ListPartsInfo {
|
||||
return ListPartsInfo{
|
||||
UploadID: lopr.UploadID,
|
||||
Bucket: lopr.Bucket,
|
||||
Object: lopr.Key,
|
||||
StorageClass: "",
|
||||
PartNumberMarker: lopr.PartNumberMarker,
|
||||
NextPartNumberMarker: lopr.NextPartNumberMarker,
|
||||
MaxParts: lopr.MaxParts,
|
||||
IsTruncated: lopr.IsTruncated,
|
||||
EncodingType: lopr.EncodingType,
|
||||
Parts: fromMinioClientObjectParts(lopr.ObjectParts),
|
||||
}
|
||||
}
|
||||
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (l *s3Objects) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, e error) {
|
||||
result, err := l.Client.ListObjectParts(bucket, object, uploadID, partNumberMarker, maxParts)
|
||||
if err != nil {
|
||||
return lpi, err
|
||||
}
|
||||
|
||||
return fromMinioClientListPartsInfo(result), nil
|
||||
}
|
||||
|
||||
// AbortMultipartUpload aborts a ongoing multipart upload
|
||||
func (l *s3Objects) AbortMultipartUpload(bucket string, object string, uploadID string) error {
|
||||
err := l.Client.AbortMultipartUpload(bucket, object, uploadID)
|
||||
return s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
// toMinioClientCompletePart converts CompletePart to minio CompletePart
|
||||
func toMinioClientCompletePart(part CompletePart) minio.CompletePart {
|
||||
return minio.CompletePart{
|
||||
ETag: part.ETag,
|
||||
PartNumber: part.PartNumber,
|
||||
}
|
||||
}
|
||||
|
||||
// toMinioClientCompleteParts converts []CompletePart to minio []CompletePart
|
||||
func toMinioClientCompleteParts(parts []CompletePart) []minio.CompletePart {
|
||||
mparts := make([]minio.CompletePart, len(parts))
|
||||
for i, part := range parts {
|
||||
mparts[i] = toMinioClientCompletePart(part)
|
||||
}
|
||||
return mparts
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (l *s3Objects) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, e error) {
|
||||
err := l.Client.CompleteMultipartUpload(bucket, object, uploadID, toMinioClientCompleteParts(uploadedParts))
|
||||
if err != nil {
|
||||
return oi, s3ToObjectError(traceError(err), bucket, object)
|
||||
}
|
||||
|
||||
return l.GetObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
// SetBucketPolicies sets policy on bucket
|
||||
func (l *s3Objects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
|
||||
if err := l.Client.PutBucketPolicy(bucket, policyInfo); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, "")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetBucketPolicies will get policy on bucket
|
||||
func (l *s3Objects) GetBucketPolicies(bucket string) (policy.BucketAccessPolicy, error) {
|
||||
policyInfo, err := l.Client.GetBucketPolicy(bucket)
|
||||
if err != nil {
|
||||
return policy.BucketAccessPolicy{}, s3ToObjectError(traceError(err), bucket, "")
|
||||
}
|
||||
return policyInfo, nil
|
||||
}
|
||||
|
||||
// DeleteBucketPolicies deletes all policies on bucket
|
||||
func (l *s3Objects) DeleteBucketPolicies(bucket string) error {
|
||||
if err := l.Client.PutBucketPolicy(bucket, policy.BucketAccessPolicy{}); err != nil {
|
||||
return s3ToObjectError(traceError(err), bucket, "")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -45,7 +45,7 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
|
||||
// Prints common server startup message. Prints credential, region and browser access.
|
||||
func printGatewayCommonMsg(apiEndpoints []string) {
|
||||
// Get saved credentials.
|
||||
cred := serverConfig.GetCredential()
|
||||
cred := globalServerConfig.GetCredential()
|
||||
|
||||
apiEndpointStr := strings.Join(apiEndpoints, " ")
|
||||
// Colorize the message and print.
|
||||
@@ -53,6 +53,8 @@ func printGatewayCommonMsg(apiEndpoints []string) {
|
||||
log.Println(colorBlue("AccessKey: ") + colorBold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
log.Println(colorBlue("SecretKey: ") + colorBold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
|
||||
log.Println(colorBlue("\nBrowser Access:"))
|
||||
log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
|
||||
if globalIsBrowserEnabled {
|
||||
log.Println(colorBlue("\nBrowser Access:"))
|
||||
log.Println(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 3), apiEndpointStr))
|
||||
}
|
||||
}
|
||||
|
||||
+107
-29
@@ -17,62 +17,140 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
type gatewayUnsupported struct{}
|
||||
// GatewayUnsupported list of unsupported call stubs for gateway.
|
||||
type GatewayUnsupported struct{}
|
||||
|
||||
// CopyObjectPart - Not implemented.
|
||||
func (a gatewayUnsupported) CopyObjectPart(srcBucket, srcObject, destBucket, destObject string, uploadID string,
|
||||
partID int, startOffset int64, length int64) (info PartInfo, err error) {
|
||||
return info, traceError(NotImplemented{})
|
||||
// ListMultipartUploads lists all multipart uploads.
|
||||
func (a GatewayUnsupported) ListMultipartUploads(bucket string, prefix string, keyMarker string, uploadIDMarker string, delimiter string, maxUploads int) (lmi ListMultipartsInfo, err error) {
|
||||
return lmi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealBucket - Not relevant.
|
||||
func (a gatewayUnsupported) HealBucket(bucket string) error {
|
||||
return traceError(NotImplemented{})
|
||||
// NewMultipartUpload upload object in multiple parts
|
||||
func (a GatewayUnsupported) NewMultipartUpload(bucket string, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
return "", errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not relevant.
|
||||
func (a gatewayUnsupported) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return nil, traceError(NotImplemented{})
|
||||
// CopyObjectPart copy part of object to uploadID for another object
|
||||
func (a GatewayUnsupported) CopyObjectPart(srcBucket, srcObject, destBucket, destObject, uploadID string, partID int, startOffset, length int64, metadata map[string]string, srcETag string) (pi PartInfo, err error) {
|
||||
return pi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not relevant.
|
||||
func (a gatewayUnsupported) HealObject(bucket, object string) (int, int, error) {
|
||||
return 0, 0, traceError(NotImplemented{})
|
||||
// PutObjectPart puts a part of object in bucket
|
||||
func (a GatewayUnsupported) PutObjectPart(bucket string, object string, uploadID string, partID int, data *hash.Reader) (pi PartInfo, err error) {
|
||||
return pi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not relevant.
|
||||
func (a gatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
// ListObjectParts returns all object parts for specified object in specified bucket
|
||||
func (a GatewayUnsupported) ListObjectParts(bucket string, object string, uploadID string, partNumberMarker int, maxParts int) (lpi ListPartsInfo, err error) {
|
||||
return lpi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListUploadsHeal - Not relevant.
|
||||
func (a gatewayUnsupported) ListUploadsHeal(bucket, prefix, marker, uploadIDMarker,
|
||||
delimiter string, maxUploads int) (lmi ListMultipartsInfo, e error) {
|
||||
return lmi, traceError(NotImplemented{})
|
||||
// AbortMultipartUpload aborts a ongoing multipart upload
|
||||
func (a GatewayUnsupported) AbortMultipartUpload(bucket string, object string, uploadID string) error {
|
||||
return errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload completes ongoing multipart upload and finalizes object
|
||||
func (a GatewayUnsupported) CompleteMultipartUpload(bucket string, object string, uploadID string, uploadedParts []CompletePart) (oi ObjectInfo, err error) {
|
||||
return oi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// SetBucketPolicies sets policy on bucket
|
||||
func (a GatewayUnsupported) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
|
||||
return errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// GetBucketPolicies will get policy on bucket
|
||||
func (a GatewayUnsupported) GetBucketPolicies(bucket string) (bal policy.BucketAccessPolicy, err error) {
|
||||
return bal, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// DeleteBucketPolicies deletes all policies on bucket
|
||||
func (a GatewayUnsupported) DeleteBucketPolicies(bucket string) error {
|
||||
return errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealBucket - Not implemented stub
|
||||
func (a GatewayUnsupported) HealBucket(bucket string, dryRun bool) ([]madmin.HealResultItem, error) {
|
||||
return nil, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListBucketsHeal - Not implemented stub
|
||||
func (a GatewayUnsupported) ListBucketsHeal() (buckets []BucketInfo, err error) {
|
||||
return nil, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// HealObject - Not implemented stub
|
||||
func (a GatewayUnsupported) HealObject(bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) {
|
||||
return h, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsV2 - Not implemented stub
|
||||
func (a GatewayUnsupported) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
|
||||
return result, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ListObjectsHeal - Not implemented stub
|
||||
func (a GatewayUnsupported) ListObjectsHeal(bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
|
||||
return loi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonListObjects - List objects anonymously
|
||||
func (a gatewayUnsupported) AnonListObjects(bucket string, prefix string, marker string, delimiter string,
|
||||
func (a GatewayUnsupported) AnonListObjects(bucket string, prefix string, marker string, delimiter string,
|
||||
maxKeys int) (loi ListObjectsInfo, err error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
return loi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonListObjectsV2 - List objects in V2 mode, anonymously
|
||||
func (a gatewayUnsupported) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
|
||||
func (a GatewayUnsupported) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int,
|
||||
fetchOwner bool, startAfter string) (loi ListObjectsV2Info, err error) {
|
||||
return loi, traceError(NotImplemented{})
|
||||
return loi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetBucketInfo - Get bucket metadata anonymously.
|
||||
func (a gatewayUnsupported) AnonGetBucketInfo(bucket string) (bi BucketInfo, err error) {
|
||||
return bi, traceError(NotImplemented{})
|
||||
func (a GatewayUnsupported) AnonGetBucketInfo(bucket string) (bi BucketInfo, err error) {
|
||||
return bi, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonPutObject creates a new object anonymously with the incoming data,
|
||||
func (a gatewayUnsupported) AnonPutObject(bucket, object string, data *hash.Reader,
|
||||
func (a GatewayUnsupported) AnonPutObject(bucket, object string, data *hash.Reader,
|
||||
metadata map[string]string) (ObjectInfo, error) {
|
||||
return ObjectInfo{}, traceError(NotImplemented{})
|
||||
return ObjectInfo{}, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetObject downloads object anonymously.
|
||||
func (a GatewayUnsupported) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
|
||||
return errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// AnonGetObjectInfo returns stat information about an object anonymously.
|
||||
func (a GatewayUnsupported) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
return objInfo, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// CopyObject copies a blob from source container to destination container.
|
||||
func (a GatewayUnsupported) CopyObject(srcBucket string, srcObject string, destBucket string, destObject string,
|
||||
metadata map[string]string, srcEtag string) (objInfo ObjectInfo, err error) {
|
||||
return objInfo, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// Locking operations
|
||||
|
||||
// ListLocks lists namespace locks held in object layer
|
||||
func (a GatewayUnsupported) ListLocks(bucket, prefix string, duration time.Duration) ([]VolumeLockInfo, error) {
|
||||
return []VolumeLockInfo{}, errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
// ClearLocks clears namespace locks held in object layer
|
||||
func (a GatewayUnsupported) ClearLocks([]VolumeLockInfo) error {
|
||||
return errors.Trace(NotImplemented{})
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
package azure
|
||||
|
||||
import (
|
||||
"encoding/xml"
|
||||
@@ -28,6 +28,9 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
// Copied from github.com/Azure/azure-sdk-for-go/storage/container.go
|
||||
@@ -112,39 +115,37 @@ func azureAnonRequest(verb, urlStr string, header http.Header) (*http.Response,
|
||||
}
|
||||
|
||||
// AnonGetBucketInfo - Get bucket metadata from azure anonymously.
|
||||
func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo BucketInfo, err error) {
|
||||
func (a *azureObjects) AnonGetBucketInfo(bucket string) (bucketInfo minio.BucketInfo, err error) {
|
||||
blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
|
||||
url, err := url.Parse(blobURL)
|
||||
if err != nil {
|
||||
return bucketInfo, azureToObjectError(traceError(err))
|
||||
return bucketInfo, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
url.RawQuery = "restype=container"
|
||||
resp, err := azureAnonRequest(httpHEAD, url.String(), nil)
|
||||
resp, err := azureAnonRequest(http.MethodHead, url.String(), nil)
|
||||
if err != nil {
|
||||
return bucketInfo, azureToObjectError(traceError(err), bucket)
|
||||
return bucketInfo, azureToObjectError(errors.Trace(err), bucket)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return bucketInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket)), bucket)
|
||||
return bucketInfo, azureToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket)), bucket)
|
||||
}
|
||||
|
||||
t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
return bucketInfo, traceError(err)
|
||||
return bucketInfo, errors.Trace(err)
|
||||
}
|
||||
|
||||
bucketInfo = BucketInfo{
|
||||
return minio.BucketInfo{
|
||||
Name: bucket,
|
||||
Created: t,
|
||||
}
|
||||
|
||||
return bucketInfo, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AnonGetObject - SendGET request without authentication.
|
||||
// This is needed when clients send GET requests on objects that can be downloaded without auth.
|
||||
func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) (err error) {
|
||||
func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) (err error) {
|
||||
h := make(http.Header)
|
||||
if length > 0 && startOffset > 0 {
|
||||
h.Add("Range", fmt.Sprintf("bytes=%d-%d", startOffset, startOffset+length-1))
|
||||
@@ -153,32 +154,32 @@ func (a *azureObjects) AnonGetObject(bucket, object string, startOffset int64, l
|
||||
}
|
||||
|
||||
blobURL := a.client.GetContainerReference(bucket).GetBlobReference(object).GetURL()
|
||||
resp, err := azureAnonRequest(httpGET, blobURL, h)
|
||||
resp, err := azureAnonRequest(http.MethodGet, blobURL, h)
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
return azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusOK {
|
||||
return azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
|
||||
return azureToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
|
||||
}
|
||||
|
||||
_, err = io.Copy(writer, resp.Body)
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
// AnonGetObjectInfo - Send HEAD request without authentication and convert the
|
||||
// result to ObjectInfo.
|
||||
func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo minio.ObjectInfo, err error) {
|
||||
blobURL := a.client.GetContainerReference(bucket).GetBlobReference(object).GetURL()
|
||||
resp, err := azureAnonRequest(httpHEAD, blobURL, nil)
|
||||
resp, err := azureAnonRequest(http.MethodHead, blobURL, nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return objInfo, azureToObjectError(traceError(anonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(minio.AnonErrToObjectErr(resp.StatusCode, bucket, object)), bucket, object)
|
||||
}
|
||||
|
||||
var contentLength int64
|
||||
@@ -186,13 +187,13 @@ func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectI
|
||||
if contentLengthStr != "" {
|
||||
contentLength, err = strconv.ParseInt(contentLengthStr, 0, 64)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(errUnexpected), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
|
||||
}
|
||||
}
|
||||
|
||||
t, err := time.Parse(time.RFC1123, resp.Header.Get("Last-Modified"))
|
||||
if err != nil {
|
||||
return objInfo, traceError(err)
|
||||
return objInfo, errors.Trace(err)
|
||||
}
|
||||
|
||||
objInfo.ModTime = t
|
||||
@@ -210,7 +211,7 @@ func (a *azureObjects) AnonGetObjectInfo(bucket, object string) (objInfo ObjectI
|
||||
}
|
||||
|
||||
// AnonListObjects - Use Azure equivalent ListBlobs.
|
||||
func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
|
||||
params := storage.ListBlobsParameters{
|
||||
Prefix: prefix,
|
||||
Marker: marker,
|
||||
@@ -225,13 +226,13 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
|
||||
blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
|
||||
url, err := url.Parse(blobURL)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
url.RawQuery = q.Encode()
|
||||
|
||||
resp, err := azureAnonRequest(httpGET, url.String(), nil)
|
||||
resp, err := azureAnonRequest(http.MethodGet, url.String(), nil)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
@@ -239,17 +240,17 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
|
||||
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
err = xml.Unmarshal(data, &listResp)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
|
||||
result.IsTruncated = listResp.NextMarker != ""
|
||||
result.NextMarker = listResp.NextMarker
|
||||
for _, object := range listResp.Blobs {
|
||||
result.Objects = append(result.Objects, ObjectInfo{
|
||||
result.Objects = append(result.Objects, minio.ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object.Name,
|
||||
ModTime: time.Time(object.Properties.LastModified),
|
||||
@@ -264,7 +265,7 @@ func (a *azureObjects) AnonListObjects(bucket, prefix, marker, delimiter string,
|
||||
}
|
||||
|
||||
// AnonListObjectsV2 - List objects in V2 mode, anonymously
|
||||
func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
|
||||
func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
|
||||
params := storage.ListBlobsParameters{
|
||||
Prefix: prefix,
|
||||
Marker: continuationToken,
|
||||
@@ -279,13 +280,13 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, deli
|
||||
blobURL := a.client.GetContainerReference(bucket).GetBlobReference("").GetURL()
|
||||
url, err := url.Parse(blobURL)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
url.RawQuery = q.Encode()
|
||||
|
||||
resp, err := http.Get(url.String())
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
@@ -293,11 +294,11 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, deli
|
||||
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
err = xml.Unmarshal(data, &listResp)
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err))
|
||||
return result, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
|
||||
// If NextMarker is not empty, this means response is truncated and NextContinuationToken should be set
|
||||
@@ -306,12 +307,12 @@ func (a *azureObjects) AnonListObjectsV2(bucket, prefix, continuationToken, deli
|
||||
result.NextContinuationToken = listResp.NextMarker
|
||||
}
|
||||
for _, object := range listResp.Blobs {
|
||||
result.Objects = append(result.Objects, ObjectInfo{
|
||||
result.Objects = append(result.Objects, minio.ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object.Name,
|
||||
ModTime: time.Time(object.Properties.LastModified),
|
||||
Size: object.Properties.ContentLength,
|
||||
ETag: canonicalizeETag(object.Properties.Etag),
|
||||
ETag: minio.CanonicalizeETag(object.Properties.Etag),
|
||||
ContentType: object.Properties.ContentType,
|
||||
ContentEncoding: object.Properties.ContentEncoding,
|
||||
})
|
||||
@@ -14,16 +14,14 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
package azure
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -36,13 +34,19 @@ import (
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio-go/pkg/policy"
|
||||
"github.com/minio/minio/pkg/auth"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
const (
|
||||
globalAzureAPIVersion = "2016-05-31"
|
||||
azureBlockSize = 100 * humanize.MiByte
|
||||
metadataObjectNameTemplate = globalMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
||||
azureS3MinPartSize = 5 * humanize.MiByte
|
||||
metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
||||
azureBackend = "azure"
|
||||
)
|
||||
|
||||
@@ -67,6 +71,9 @@ ENVIRONMENT VARIABLES:
|
||||
BROWSER:
|
||||
MINIO_BROWSER: To disable web browser access, set this value to "off".
|
||||
|
||||
UPDATE:
|
||||
MINIO_UPDATE: To turn off in-place upgrades, set this value to "off".
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend.
|
||||
$ export MINIO_ACCESS_KEY=azureaccountname
|
||||
@@ -80,12 +87,11 @@ EXAMPLES:
|
||||
|
||||
`
|
||||
|
||||
MustRegisterGatewayCommand(cli.Command{
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
Name: azureBackend,
|
||||
Usage: "Microsoft Azure Blob Storage.",
|
||||
Action: azureGatewayMain,
|
||||
CustomHelpTemplate: azureGatewayTemplate,
|
||||
Flags: append(serverFlags, globalFlags...),
|
||||
HideHelpCommand: true,
|
||||
})
|
||||
}
|
||||
@@ -95,24 +101,53 @@ func azureGatewayMain(ctx *cli.Context) {
|
||||
// Validate gateway arguments.
|
||||
host := ctx.Args().First()
|
||||
// Validate gateway arguments.
|
||||
fatalIf(validateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||
minio.FatalIf(minio.ValidateGatewayArguments(ctx.GlobalString("address"), host), "Invalid argument")
|
||||
|
||||
startGateway(ctx, &AzureGateway{host})
|
||||
minio.StartGateway(ctx, &Azure{host})
|
||||
}
|
||||
|
||||
// AzureGateway implements Gateway.
|
||||
type AzureGateway struct {
|
||||
// Azure implements Gateway.
|
||||
type Azure struct {
|
||||
host string
|
||||
}
|
||||
|
||||
// Name implements Gateway interface.
|
||||
func (g *AzureGateway) Name() string {
|
||||
func (g *Azure) Name() string {
|
||||
return azureBackend
|
||||
}
|
||||
|
||||
// NewGatewayLayer initializes azure blob storage client and returns AzureObjects.
|
||||
func (g *AzureGateway) NewGatewayLayer() (GatewayLayer, error) {
|
||||
return newAzureLayer(g.host)
|
||||
func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.GatewayLayer, error) {
|
||||
var err error
|
||||
var endpoint = storage.DefaultBaseURL
|
||||
var secure = true
|
||||
|
||||
// If user provided some parameters
|
||||
if g.host != "" {
|
||||
endpoint, secure, err = minio.ParseGatewayEndpoint(g.host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if endpoint == fmt.Sprintf("%s.blob.%s", creds.AccessKey, storage.DefaultBaseURL) {
|
||||
// If, by mistake, user provides endpoint as accountname.blob.core.windows.net
|
||||
endpoint = storage.DefaultBaseURL
|
||||
}
|
||||
c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
|
||||
if err != nil {
|
||||
return &azureObjects{}, err
|
||||
}
|
||||
c.HTTPClient = &http.Client{Transport: minio.NewCustomHTTPTransport()}
|
||||
|
||||
return &azureObjects{
|
||||
client: c.GetBlobService(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Production - Azure gateway is production ready.
|
||||
func (g *Azure) Production() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// s3MetaToAzureProperties converts metadata meant for S3 PUT/COPY
|
||||
@@ -133,7 +168,7 @@ func s3MetaToAzureProperties(s3Metadata map[string]string) (storage.BlobMetadata
|
||||
storage.BlobProperties, error) {
|
||||
for k := range s3Metadata {
|
||||
if strings.Contains(k, "--") {
|
||||
return storage.BlobMetadata{}, storage.BlobProperties{}, traceError(UnsupportedMetadata{})
|
||||
return storage.BlobMetadata{}, storage.BlobProperties{}, errors.Trace(minio.UnsupportedMetadata{})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,7 +273,7 @@ func azurePropertiesToS3Meta(meta storage.BlobMetadata, props storage.BlobProper
|
||||
|
||||
// azureObjects - Implements Object layer for Azure blob storage.
|
||||
type azureObjects struct {
|
||||
gatewayUnsupported
|
||||
minio.GatewayUnsupported
|
||||
client storage.BlobStorageClient // Azure sdk client
|
||||
}
|
||||
|
||||
@@ -248,15 +283,15 @@ func azureToObjectError(err error, params ...string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
e, ok := err.(*Error)
|
||||
e, ok := err.(*errors.Error)
|
||||
if !ok {
|
||||
// Code should be fixed if this function is called without doing traceError()
|
||||
// Code should be fixed if this function is called without doing errors.Trace()
|
||||
// Else handling different situations in this function makes this function complicated.
|
||||
errorIf(err, "Expected type *Error")
|
||||
minio.ErrorIf(err, "Expected type *Error")
|
||||
return err
|
||||
}
|
||||
|
||||
err = e.e
|
||||
err = e.Cause
|
||||
bucket := ""
|
||||
object := ""
|
||||
if len(params) >= 1 {
|
||||
@@ -275,26 +310,29 @@ func azureToObjectError(err error, params ...string) error {
|
||||
|
||||
switch azureErr.Code {
|
||||
case "ContainerAlreadyExists":
|
||||
err = BucketExists{Bucket: bucket}
|
||||
err = minio.BucketExists{Bucket: bucket}
|
||||
case "InvalidResourceName":
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
err = minio.BucketNameInvalid{Bucket: bucket}
|
||||
case "RequestBodyTooLarge":
|
||||
err = PartTooBig{}
|
||||
err = minio.PartTooBig{}
|
||||
case "InvalidMetadata":
|
||||
err = UnsupportedMetadata{}
|
||||
err = minio.UnsupportedMetadata{}
|
||||
default:
|
||||
switch azureErr.StatusCode {
|
||||
case http.StatusNotFound:
|
||||
if object != "" {
|
||||
err = ObjectNotFound{bucket, object}
|
||||
err = minio.ObjectNotFound{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
}
|
||||
} else {
|
||||
err = BucketNotFound{Bucket: bucket}
|
||||
err = minio.BucketNotFound{Bucket: bucket}
|
||||
}
|
||||
case http.StatusBadRequest:
|
||||
err = BucketNameInvalid{Bucket: bucket}
|
||||
err = minio.BucketNameInvalid{Bucket: bucket}
|
||||
}
|
||||
}
|
||||
e.e = err
|
||||
e.Cause = err
|
||||
return e
|
||||
}
|
||||
|
||||
@@ -316,11 +354,15 @@ func mustGetAzureUploadID() string {
|
||||
// checkAzureUploadID - returns error in case of given string is upload ID.
|
||||
func checkAzureUploadID(uploadID string) (err error) {
|
||||
if len(uploadID) != 16 {
|
||||
return traceError(MalformedUploadID{uploadID})
|
||||
return errors.Trace(minio.MalformedUploadID{
|
||||
UploadID: uploadID,
|
||||
})
|
||||
}
|
||||
|
||||
if _, err = hex.DecodeString(uploadID); err != nil {
|
||||
return traceError(MalformedUploadID{uploadID})
|
||||
return errors.Trace(minio.MalformedUploadID{
|
||||
UploadID: uploadID,
|
||||
})
|
||||
}
|
||||
|
||||
return nil
|
||||
@@ -360,32 +402,6 @@ func azureParseBlockID(blockID string) (partID, subPartNumber int, uploadID, md5
|
||||
return
|
||||
}
|
||||
|
||||
// Inits azure blob storage client and returns AzureObjects.
|
||||
func newAzureLayer(host string) (GatewayLayer, error) {
|
||||
var err error
|
||||
var endpoint = storage.DefaultBaseURL
|
||||
var secure = true
|
||||
|
||||
// If user provided some parameters
|
||||
if host != "" {
|
||||
endpoint, secure, err = parseGatewayEndpoint(host)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
creds := serverConfig.GetCredential()
|
||||
c, err := storage.NewClient(creds.AccessKey, creds.SecretKey, endpoint, globalAzureAPIVersion, secure)
|
||||
if err != nil {
|
||||
return &azureObjects{}, err
|
||||
}
|
||||
c.HTTPClient = &http.Client{Transport: newCustomHTTPTransport()}
|
||||
|
||||
return &azureObjects{
|
||||
client: c.GetBlobService(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Shutdown - save any gateway metadata to disk
|
||||
// if necessary and reload upon next restart.
|
||||
func (a *azureObjects) Shutdown() error {
|
||||
@@ -393,7 +409,7 @@ func (a *azureObjects) Shutdown() error {
|
||||
}
|
||||
|
||||
// StorageInfo - Not relevant to Azure backend.
|
||||
func (a *azureObjects) StorageInfo() (si StorageInfo) {
|
||||
func (a *azureObjects) StorageInfo() (si minio.StorageInfo) {
|
||||
return si
|
||||
}
|
||||
|
||||
@@ -403,17 +419,17 @@ func (a *azureObjects) MakeBucketWithLocation(bucket, location string) error {
|
||||
err := container.Create(&storage.CreateContainerOptions{
|
||||
Access: storage.ContainerAccessTypePrivate,
|
||||
})
|
||||
return azureToObjectError(traceError(err), bucket)
|
||||
return azureToObjectError(errors.Trace(err), bucket)
|
||||
}
|
||||
|
||||
// GetBucketInfo - Get bucket metadata..
|
||||
func (a *azureObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
func (a *azureObjects) GetBucketInfo(bucket string) (bi minio.BucketInfo, e error) {
|
||||
// Verify if bucket (container-name) is valid.
|
||||
// IsValidBucketName has same restrictions as container names mentioned
|
||||
// in azure documentation, so we will simply use the same function here.
|
||||
// Ref - https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata
|
||||
if !IsValidBucketName(bucket) {
|
||||
return bi, traceError(BucketNameInvalid{Bucket: bucket})
|
||||
if !minio.IsValidBucketName(bucket) {
|
||||
return bi, errors.Trace(minio.BucketNameInvalid{Bucket: bucket})
|
||||
}
|
||||
|
||||
// Azure does not have an equivalent call, hence use
|
||||
@@ -422,34 +438,34 @@ func (a *azureObjects) GetBucketInfo(bucket string) (bi BucketInfo, e error) {
|
||||
Prefix: bucket,
|
||||
})
|
||||
if err != nil {
|
||||
return bi, azureToObjectError(traceError(err), bucket)
|
||||
return bi, azureToObjectError(errors.Trace(err), bucket)
|
||||
}
|
||||
for _, container := range resp.Containers {
|
||||
if container.Name == bucket {
|
||||
t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
|
||||
if e == nil {
|
||||
return BucketInfo{
|
||||
return minio.BucketInfo{
|
||||
Name: bucket,
|
||||
Created: t,
|
||||
}, nil
|
||||
} // else continue
|
||||
}
|
||||
}
|
||||
return bi, traceError(BucketNotFound{Bucket: bucket})
|
||||
return bi, errors.Trace(minio.BucketNotFound{Bucket: bucket})
|
||||
}
|
||||
|
||||
// ListBuckets - Lists all azure containers, uses Azure equivalent ListContainers.
|
||||
func (a *azureObjects) ListBuckets() (buckets []BucketInfo, err error) {
|
||||
func (a *azureObjects) ListBuckets() (buckets []minio.BucketInfo, err error) {
|
||||
resp, err := a.client.ListContainers(storage.ListContainersParameters{})
|
||||
if err != nil {
|
||||
return nil, azureToObjectError(traceError(err))
|
||||
return nil, azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
for _, container := range resp.Containers {
|
||||
t, e := time.Parse(time.RFC1123, container.Properties.LastModified)
|
||||
if e != nil {
|
||||
return nil, traceError(e)
|
||||
return nil, errors.Trace(e)
|
||||
}
|
||||
buckets = append(buckets, BucketInfo{
|
||||
buckets = append(buckets, minio.BucketInfo{
|
||||
Name: container.Name,
|
||||
Created: t,
|
||||
})
|
||||
@@ -460,13 +476,13 @@ func (a *azureObjects) ListBuckets() (buckets []BucketInfo, err error) {
|
||||
// DeleteBucket - delete a container on azure, uses Azure equivalent DeleteContainer.
|
||||
func (a *azureObjects) DeleteBucket(bucket string) error {
|
||||
container := a.client.GetContainerReference(bucket)
|
||||
return azureToObjectError(traceError(container.Delete(nil)), bucket)
|
||||
return azureToObjectError(errors.Trace(container.Delete(nil)), bucket)
|
||||
}
|
||||
|
||||
// ListObjects - lists all blobs on azure with in a container filtered by prefix
|
||||
// and marker, uses Azure equivalent ListBlobs.
|
||||
func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) {
|
||||
var objects []ObjectInfo
|
||||
func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result minio.ListObjectsInfo, err error) {
|
||||
var objects []minio.ObjectInfo
|
||||
var prefixes []string
|
||||
container := a.client.GetContainerReference(bucket)
|
||||
for len(objects) == 0 && len(prefixes) == 0 {
|
||||
@@ -477,19 +493,19 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max
|
||||
MaxResults: uint(maxKeys),
|
||||
})
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err), bucket, prefix)
|
||||
return result, azureToObjectError(errors.Trace(err), bucket, prefix)
|
||||
}
|
||||
|
||||
for _, object := range resp.Blobs {
|
||||
if strings.HasPrefix(object.Name, globalMinioSysTmp) {
|
||||
if strings.HasPrefix(object.Name, minio.GatewayMinioSysTmp) {
|
||||
continue
|
||||
}
|
||||
objects = append(objects, ObjectInfo{
|
||||
objects = append(objects, minio.ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object.Name,
|
||||
ModTime: time.Time(object.Properties.LastModified),
|
||||
Size: object.Properties.ContentLength,
|
||||
ETag: toS3ETag(object.Properties.Etag),
|
||||
ETag: minio.ToS3ETag(object.Properties.Etag),
|
||||
ContentType: object.Properties.ContentType,
|
||||
ContentEncoding: object.Properties.ContentEncoding,
|
||||
})
|
||||
@@ -497,7 +513,7 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max
|
||||
|
||||
// Remove minio.sys.tmp prefix.
|
||||
for _, prefix := range resp.BlobPrefixes {
|
||||
if prefix != globalMinioSysTmp {
|
||||
if prefix != minio.GatewayMinioSysTmp {
|
||||
prefixes = append(prefixes, prefix)
|
||||
}
|
||||
}
|
||||
@@ -516,13 +532,13 @@ func (a *azureObjects) ListObjects(bucket, prefix, marker, delimiter string, max
|
||||
}
|
||||
|
||||
// ListObjectsV2 - list all blobs in Azure bucket filtered by prefix
|
||||
func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) {
|
||||
func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result minio.ListObjectsV2Info, err error) {
|
||||
marker := continuationToken
|
||||
if startAfter != "" {
|
||||
marker = startAfter
|
||||
}
|
||||
|
||||
var resultV1 ListObjectsInfo
|
||||
var resultV1 minio.ListObjectsInfo
|
||||
resultV1, err = a.ListObjects(bucket, prefix, marker, delimiter, maxKeys)
|
||||
if err != nil {
|
||||
return result, err
|
||||
@@ -542,10 +558,10 @@ func (a *azureObjects) ListObjectsV2(bucket, prefix, continuationToken, delimite
|
||||
//
|
||||
// startOffset indicates the starting read location of the object.
|
||||
// length indicates the total length of the object.
|
||||
func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
func (a *azureObjects) GetObject(bucket, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
|
||||
// startOffset cannot be negative.
|
||||
if startOffset < 0 {
|
||||
return toObjectErr(traceError(errUnexpected), bucket, object)
|
||||
return azureToObjectError(errors.Trace(minio.InvalidRange{}), bucket, object)
|
||||
}
|
||||
|
||||
blobRange := &storage.BlobRange{Start: uint64(startOffset)}
|
||||
@@ -564,39 +580,37 @@ func (a *azureObjects) GetObject(bucket, object string, startOffset int64, lengt
|
||||
})
|
||||
}
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
return azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
_, err = io.Copy(writer, rc)
|
||||
rc.Close()
|
||||
return traceError(err)
|
||||
return errors.Trace(err)
|
||||
}
|
||||
|
||||
// GetObjectInfo - reads blob metadata properties and replies back ObjectInfo,
|
||||
// GetObjectInfo - reads blob metadata properties and replies back minio.ObjectInfo,
|
||||
// uses zure equivalent GetBlobProperties.
|
||||
func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) GetObjectInfo(bucket, object string) (objInfo minio.ObjectInfo, err error) {
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
err = blob.GetProperties(nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
meta := azurePropertiesToS3Meta(blob.Metadata, blob.Properties)
|
||||
objInfo = ObjectInfo{
|
||||
return minio.ObjectInfo{
|
||||
Bucket: bucket,
|
||||
UserDefined: meta,
|
||||
ETag: toS3ETag(blob.Properties.Etag),
|
||||
UserDefined: azurePropertiesToS3Meta(blob.Metadata, blob.Properties),
|
||||
ETag: minio.ToS3ETag(blob.Properties.Etag),
|
||||
ModTime: time.Time(blob.Properties.LastModified),
|
||||
Name: object,
|
||||
Size: blob.Properties.ContentLength,
|
||||
ContentType: blob.Properties.ContentType,
|
||||
ContentEncoding: blob.Properties.ContentEncoding,
|
||||
}
|
||||
return objInfo, nil
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PutObject - Create a new blob with the incoming data,
|
||||
// uses Azure equivalent CreateBlockBlobFromReader.
|
||||
func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metadata map[string]string) (objInfo minio.ObjectInfo, err error) {
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
blob.Metadata, blob.Properties, err = s3MetaToAzureProperties(metadata)
|
||||
if err != nil {
|
||||
@@ -604,14 +618,14 @@ func (a *azureObjects) PutObject(bucket, object string, data *hash.Reader, metad
|
||||
}
|
||||
err = blob.CreateBlockBlobFromReader(data, nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
return a.GetObjectInfo(bucket, object)
|
||||
}
|
||||
|
||||
// CopyObject - Copies a blob from source container to destination container.
|
||||
// Uses Azure equivalent CopyBlob API.
|
||||
func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject string, metadata map[string]string, srcEtag string) (objInfo minio.ObjectInfo, err error) {
|
||||
srcBlobURL := a.client.GetContainerReference(srcBucket).GetBlobReference(srcObject).GetURL()
|
||||
destBlob := a.client.GetContainerReference(destBucket).GetBlobReference(destObject)
|
||||
azureMeta, props, err := s3MetaToAzureProperties(metadata)
|
||||
@@ -621,12 +635,12 @@ func (a *azureObjects) CopyObject(srcBucket, srcObject, destBucket, destObject s
|
||||
destBlob.Metadata = azureMeta
|
||||
err = destBlob.Copy(srcBlobURL, nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject)
|
||||
}
|
||||
destBlob.Properties = props
|
||||
err = destBlob.SetProperties(nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), srcBucket, srcObject)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), srcBucket, srcObject)
|
||||
}
|
||||
return a.GetObjectInfo(destBucket, destObject)
|
||||
}
|
||||
@@ -637,13 +651,13 @@ func (a *azureObjects) DeleteObject(bucket, object string) error {
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
err := blob.Delete(nil)
|
||||
if err != nil {
|
||||
return azureToObjectError(traceError(err), bucket, object)
|
||||
return azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ListMultipartUploads - It's decided not to support List Multipart Uploads, hence returning empty result.
|
||||
func (a *azureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
|
||||
func (a *azureObjects) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result minio.ListMultipartsInfo, err error) {
|
||||
// It's decided not to support List Multipart Uploads, hence returning empty result.
|
||||
return result, nil
|
||||
}
|
||||
@@ -661,10 +675,15 @@ func (a *azureObjects) checkUploadIDExists(bucketName, objectName, uploadID stri
|
||||
blob := a.client.GetContainerReference(bucketName).GetBlobReference(
|
||||
getAzureMetadataObjectName(objectName, uploadID))
|
||||
err = blob.GetMetadata(nil)
|
||||
err = azureToObjectError(traceError(err), bucketName, objectName)
|
||||
oerr := ObjectNotFound{bucketName, objectName}
|
||||
if errorCause(err) == oerr {
|
||||
err = traceError(InvalidUploadID{})
|
||||
err = azureToObjectError(errors.Trace(err), bucketName, objectName)
|
||||
oerr := minio.ObjectNotFound{
|
||||
Bucket: bucketName,
|
||||
Object: objectName,
|
||||
}
|
||||
if errors.Cause(err) == oerr {
|
||||
err = errors.Trace(minio.InvalidUploadID{
|
||||
UploadID: uploadID,
|
||||
})
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -673,26 +692,26 @@ func (a *azureObjects) checkUploadIDExists(bucketName, objectName, uploadID stri
|
||||
func (a *azureObjects) NewMultipartUpload(bucket, object string, metadata map[string]string) (uploadID string, err error) {
|
||||
uploadID = mustGetAzureUploadID()
|
||||
if err = a.checkUploadIDExists(bucket, object, uploadID); err == nil {
|
||||
return "", traceError(errors.New("Upload ID name collision"))
|
||||
return "", errors.Trace(fmt.Errorf("Upload ID name collision"))
|
||||
}
|
||||
metadataObject := getAzureMetadataObjectName(object, uploadID)
|
||||
|
||||
var jsonData []byte
|
||||
if jsonData, err = json.Marshal(azureMultipartMetadata{Name: object, Metadata: metadata}); err != nil {
|
||||
return "", traceError(err)
|
||||
return "", errors.Trace(err)
|
||||
}
|
||||
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
|
||||
err = blob.CreateBlockBlobFromReader(bytes.NewBuffer(jsonData), nil)
|
||||
if err != nil {
|
||||
return "", azureToObjectError(traceError(err), bucket, metadataObject)
|
||||
return "", azureToObjectError(errors.Trace(err), bucket, metadataObject)
|
||||
}
|
||||
|
||||
return uploadID, nil
|
||||
}
|
||||
|
||||
// PutObjectPart - Use Azure equivalent PutBlockWithLength.
|
||||
func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (info PartInfo, err error) {
|
||||
func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int, data *hash.Reader) (info minio.PartInfo, err error) {
|
||||
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
|
||||
return info, err
|
||||
}
|
||||
@@ -703,7 +722,7 @@ func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int
|
||||
|
||||
etag := data.MD5HexString()
|
||||
if etag == "" {
|
||||
etag = genETag()
|
||||
etag = minio.GenETag()
|
||||
}
|
||||
|
||||
subPartSize, subPartNumber := int64(azureBlockSize), 1
|
||||
@@ -721,20 +740,20 @@ func (a *azureObjects) PutObjectPart(bucket, object, uploadID string, partID int
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
err = blob.PutBlockWithLength(id, uint64(subPartSize), io.LimitReader(data, subPartSize), nil)
|
||||
if err != nil {
|
||||
return info, azureToObjectError(traceError(err), bucket, object)
|
||||
return info, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
subPartNumber++
|
||||
}
|
||||
|
||||
info.PartNumber = partID
|
||||
info.ETag = etag
|
||||
info.LastModified = UTCNow()
|
||||
info.LastModified = minio.UTCNow()
|
||||
info.Size = data.Size()
|
||||
return info, nil
|
||||
}
|
||||
|
||||
// ListObjectParts - Use Azure equivalent GetBlockList.
|
||||
func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListPartsInfo, err error) {
|
||||
func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result minio.ListPartsInfo, err error) {
|
||||
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
|
||||
return result, err
|
||||
}
|
||||
@@ -746,24 +765,29 @@ func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumb
|
||||
|
||||
objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
|
||||
azureErr, ok := err.(storage.AzureStorageServiceError)
|
||||
if ok && azureErr.StatusCode == http.StatusNotFound {
|
||||
// If no parts are uploaded yet then we return empty list.
|
||||
return result, nil
|
||||
}
|
||||
if err != nil {
|
||||
return result, azureToObjectError(traceError(err), bucket, object)
|
||||
return result, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
// Build a sorted list of parts and return the requested entries.
|
||||
partsMap := make(map[int]PartInfo)
|
||||
partsMap := make(map[int]minio.PartInfo)
|
||||
for _, block := range resp.UncommittedBlocks {
|
||||
var partNumber int
|
||||
var parsedUploadID string
|
||||
var md5Hex string
|
||||
if partNumber, _, parsedUploadID, md5Hex, err = azureParseBlockID(block.Name); err != nil {
|
||||
return result, azureToObjectError(traceError(errUnexpected), bucket, object)
|
||||
return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
|
||||
}
|
||||
if parsedUploadID != uploadID {
|
||||
continue
|
||||
}
|
||||
part, ok := partsMap[partNumber]
|
||||
if !ok {
|
||||
partsMap[partNumber] = PartInfo{
|
||||
partsMap[partNumber] = minio.PartInfo{
|
||||
PartNumber: partNumber,
|
||||
Size: block.Size,
|
||||
ETag: md5Hex,
|
||||
@@ -773,12 +797,12 @@ func (a *azureObjects) ListObjectParts(bucket, object, uploadID string, partNumb
|
||||
if part.ETag != md5Hex {
|
||||
// If two parts of same partNumber were uploaded with different contents
|
||||
// return error as we won't be able to decide which the latest part is.
|
||||
return result, azureToObjectError(traceError(errUnexpected), bucket, object)
|
||||
return result, azureToObjectError(errors.Trace(fmt.Errorf("Unexpected error")), bucket, object)
|
||||
}
|
||||
part.Size += block.Size
|
||||
partsMap[partNumber] = part
|
||||
}
|
||||
var parts []PartInfo
|
||||
var parts []minio.PartInfo
|
||||
for _, part := range partsMap {
|
||||
parts = append(parts, part)
|
||||
}
|
||||
@@ -826,7 +850,7 @@ func (a *azureObjects) AbortMultipartUpload(bucket, object, uploadID string) (er
|
||||
}
|
||||
|
||||
// CompleteMultipartUpload - Use Azure equivalent PutBlockList.
|
||||
func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []CompletePart) (objInfo ObjectInfo, err error) {
|
||||
func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string, uploadedParts []minio.CompletePart) (objInfo minio.ObjectInfo, err error) {
|
||||
metadataObject := getAzureMetadataObjectName(object, uploadID)
|
||||
if err = a.checkUploadIDExists(bucket, object, uploadID); err != nil {
|
||||
return objInfo, err
|
||||
@@ -839,12 +863,12 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
var metadataReader io.Reader
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
|
||||
if metadataReader, err = blob.Get(nil); err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, metadataObject)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject)
|
||||
}
|
||||
|
||||
var metadata azureMultipartMetadata
|
||||
if err = json.NewDecoder(metadataReader).Decode(&metadata); err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, metadataObject)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, metadataObject)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
@@ -854,13 +878,13 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
|
||||
blob := a.client.GetContainerReference(bucket).GetBlobReference(metadataObject)
|
||||
derr := blob.Delete(nil)
|
||||
errorIf(derr, "unable to remove meta data object for upload ID %s", uploadID)
|
||||
minio.ErrorIf(derr, "unable to remove meta data object for upload ID %s", uploadID)
|
||||
}()
|
||||
|
||||
objBlob := a.client.GetContainerReference(bucket).GetBlobReference(object)
|
||||
resp, err := objBlob.GetBlockList(storage.BlockListTypeUncommitted, nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
getBlocks := func(partNumber int, etag string) (blocks []storage.Block, size int64, err error) {
|
||||
@@ -883,7 +907,7 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
}
|
||||
|
||||
if len(blocks) == 0 {
|
||||
return nil, 0, InvalidPart{}
|
||||
return nil, 0, minio.InvalidPart{}
|
||||
}
|
||||
|
||||
return blocks, size, nil
|
||||
@@ -896,7 +920,7 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
var size int64
|
||||
blocks, size, err = getBlocks(part.PartNumber, part.ETag)
|
||||
if err != nil {
|
||||
return objInfo, traceError(err)
|
||||
return objInfo, errors.Trace(err)
|
||||
}
|
||||
|
||||
allBlocks = append(allBlocks, blocks...)
|
||||
@@ -905,8 +929,8 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
|
||||
// Error out if parts except last part sizing < 5MiB.
|
||||
for i, size := range partSizes[:len(partSizes)-1] {
|
||||
if size < globalMinPartSize {
|
||||
return objInfo, traceError(PartTooSmall{
|
||||
if size < azureS3MinPartSize {
|
||||
return objInfo, errors.Trace(minio.PartTooSmall{
|
||||
PartNumber: uploadedParts[i].PartNumber,
|
||||
PartSize: size,
|
||||
PartETag: uploadedParts[i].ETag,
|
||||
@@ -916,7 +940,7 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
|
||||
err = objBlob.PutBlockList(allBlocks, nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
if len(metadata.Metadata) > 0 {
|
||||
objBlob.Metadata, objBlob.Properties, err = s3MetaToAzureProperties(metadata.Metadata)
|
||||
@@ -925,11 +949,11 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
}
|
||||
err = objBlob.SetProperties(nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
err = objBlob.SetMetadata(nil)
|
||||
if err != nil {
|
||||
return objInfo, azureToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, azureToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
}
|
||||
return a.GetObjectInfo(bucket, object)
|
||||
@@ -942,23 +966,23 @@ func (a *azureObjects) CompleteMultipartUpload(bucket, object, uploadID string,
|
||||
// As the common denominator for minio and azure is readonly and none, we support
|
||||
// these two policies at the bucket level.
|
||||
func (a *azureObjects) SetBucketPolicies(bucket string, policyInfo policy.BucketAccessPolicy) error {
|
||||
var policies []BucketAccessPolicy
|
||||
var policies []minio.BucketAccessPolicy
|
||||
|
||||
for prefix, policy := range policy.GetPolicies(policyInfo.Statements, bucket) {
|
||||
policies = append(policies, BucketAccessPolicy{
|
||||
policies = append(policies, minio.BucketAccessPolicy{
|
||||
Prefix: prefix,
|
||||
Policy: policy,
|
||||
})
|
||||
}
|
||||
prefix := bucket + "/*" // For all objects inside the bucket.
|
||||
if len(policies) != 1 {
|
||||
return traceError(NotImplemented{})
|
||||
return errors.Trace(minio.NotImplemented{})
|
||||
}
|
||||
if policies[0].Prefix != prefix {
|
||||
return traceError(NotImplemented{})
|
||||
return errors.Trace(minio.NotImplemented{})
|
||||
}
|
||||
if policies[0].Policy != policy.BucketPolicyReadOnly {
|
||||
return traceError(NotImplemented{})
|
||||
return errors.Trace(minio.NotImplemented{})
|
||||
}
|
||||
perm := storage.ContainerPermissions{
|
||||
AccessType: storage.ContainerAccessTypeContainer,
|
||||
@@ -966,7 +990,7 @@ func (a *azureObjects) SetBucketPolicies(bucket string, policyInfo policy.Bucket
|
||||
}
|
||||
container := a.client.GetContainerReference(bucket)
|
||||
err := container.SetPermissions(perm, nil)
|
||||
return azureToObjectError(traceError(err), bucket)
|
||||
return azureToObjectError(errors.Trace(err), bucket)
|
||||
}
|
||||
|
||||
// GetBucketPolicies - Get the container ACL and convert it to canonical []bucketAccessPolicy
|
||||
@@ -975,15 +999,15 @@ func (a *azureObjects) GetBucketPolicies(bucket string) (policy.BucketAccessPoli
|
||||
container := a.client.GetContainerReference(bucket)
|
||||
perm, err := container.GetPermissions(nil)
|
||||
if err != nil {
|
||||
return policy.BucketAccessPolicy{}, azureToObjectError(traceError(err), bucket)
|
||||
return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(err), bucket)
|
||||
}
|
||||
switch perm.AccessType {
|
||||
case storage.ContainerAccessTypePrivate:
|
||||
return policy.BucketAccessPolicy{}, traceError(PolicyNotFound{Bucket: bucket})
|
||||
return policy.BucketAccessPolicy{}, errors.Trace(minio.PolicyNotFound{Bucket: bucket})
|
||||
case storage.ContainerAccessTypeContainer:
|
||||
policyInfo.Statements = policy.SetPolicy(policyInfo.Statements, policy.BucketPolicyReadOnly, bucket, "")
|
||||
default:
|
||||
return policy.BucketAccessPolicy{}, azureToObjectError(traceError(NotImplemented{}))
|
||||
return policy.BucketAccessPolicy{}, azureToObjectError(errors.Trace(minio.NotImplemented{}))
|
||||
}
|
||||
return policyInfo, nil
|
||||
}
|
||||
@@ -996,5 +1020,5 @@ func (a *azureObjects) DeleteBucketPolicies(bucket string) error {
|
||||
}
|
||||
container := a.client.GetContainerReference(bucket)
|
||||
err := container.SetPermissions(perm, nil)
|
||||
return azureToObjectError(traceError(err))
|
||||
return azureToObjectError(errors.Trace(err))
|
||||
}
|
||||
@@ -14,15 +14,18 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
package azure
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/storage"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
)
|
||||
|
||||
// Test canonical metadata.
|
||||
@@ -30,6 +33,10 @@ func TestS3MetaToAzureProperties(t *testing.T) {
|
||||
headers := map[string]string{
|
||||
"accept-encoding": "gzip",
|
||||
"content-encoding": "gzip",
|
||||
"cache-control": "age: 3600",
|
||||
"content-disposition": "dummy",
|
||||
"content-length": "10",
|
||||
"content-type": "application/javascript",
|
||||
"X-Amz-Meta-Hdr": "value",
|
||||
"X-Amz-Meta-X_test_key": "value",
|
||||
"X-Amz-Meta-X__test__key": "value",
|
||||
@@ -60,8 +67,8 @@ func TestS3MetaToAzureProperties(t *testing.T) {
|
||||
"invalid--meta": "value",
|
||||
}
|
||||
_, _, err = s3MetaToAzureProperties(headers)
|
||||
if err = errorCause(err); err != nil {
|
||||
if _, ok := err.(UnsupportedMetadata); !ok {
|
||||
if err = errors.Cause(err); err != nil {
|
||||
if _, ok := err.(minio.UnsupportedMetadata); !ok {
|
||||
t.Fatalf("Test failed with unexpected error %s, expected UnsupportedMetadata", err)
|
||||
}
|
||||
}
|
||||
@@ -100,8 +107,21 @@ func TestAzurePropertiesToS3Meta(t *testing.T) {
|
||||
"X-Amz-Meta-X-Amz-Key": "hu3ZSqtqwn+aL4V2VhAeov4i+bG3KyCtRMSXQFRHXOk=",
|
||||
"X-Amz-Meta-X-Amz-Matdesc": "{}",
|
||||
"X-Amz-Meta-X-Amz-Iv": "eWmyryl8kq+EVnnsE7jpOg==",
|
||||
"Cache-Control": "max-age: 3600",
|
||||
"Content-Disposition": "dummy",
|
||||
"Content-Encoding": "gzip",
|
||||
"Content-Length": "10",
|
||||
"Content-MD5": "base64-md5",
|
||||
"Content-Type": "application/javascript",
|
||||
}
|
||||
actualMeta := azurePropertiesToS3Meta(metadata, storage.BlobProperties{})
|
||||
actualMeta := azurePropertiesToS3Meta(metadata, storage.BlobProperties{
|
||||
CacheControl: "max-age: 3600",
|
||||
ContentDisposition: "dummy",
|
||||
ContentEncoding: "gzip",
|
||||
ContentLength: 10,
|
||||
ContentMD5: "base64-md5",
|
||||
ContentType: "application/javascript",
|
||||
})
|
||||
if !reflect.DeepEqual(actualMeta, expectedMeta) {
|
||||
t.Fatalf("Test failed, expected %#v, got %#v", expectedMeta, actualMeta)
|
||||
}
|
||||
@@ -118,43 +138,57 @@ func TestAzureToObjectError(t *testing.T) {
|
||||
nil, nil, "", "",
|
||||
},
|
||||
{
|
||||
traceError(errUnexpected), errUnexpected, "", "",
|
||||
errors.Trace(fmt.Errorf("Non azure error")),
|
||||
fmt.Errorf("Non azure error"), "", "",
|
||||
},
|
||||
{
|
||||
traceError(errUnexpected), traceError(errUnexpected), "", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
storage.AzureStorageServiceError{
|
||||
Code: "ContainerAlreadyExists",
|
||||
}), BucketExists{Bucket: "bucket"}, "bucket", "",
|
||||
}, storage.AzureStorageServiceError{
|
||||
Code: "ContainerAlreadyExists",
|
||||
}, "bucket", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
errors.Trace(storage.AzureStorageServiceError{
|
||||
Code: "ContainerAlreadyExists",
|
||||
}), minio.BucketExists{Bucket: "bucket"}, "bucket", "",
|
||||
},
|
||||
{
|
||||
errors.Trace(storage.AzureStorageServiceError{
|
||||
Code: "InvalidResourceName",
|
||||
}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
|
||||
}), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
errors.Trace(storage.AzureStorageServiceError{
|
||||
Code: "RequestBodyTooLarge",
|
||||
}), minio.PartTooBig{}, "", "",
|
||||
},
|
||||
{
|
||||
errors.Trace(storage.AzureStorageServiceError{
|
||||
Code: "InvalidMetadata",
|
||||
}), minio.UnsupportedMetadata{}, "", "",
|
||||
},
|
||||
{
|
||||
errors.Trace(storage.AzureStorageServiceError{
|
||||
StatusCode: http.StatusNotFound,
|
||||
}), ObjectNotFound{
|
||||
}), minio.ObjectNotFound{
|
||||
Bucket: "bucket",
|
||||
Object: "object",
|
||||
}, "bucket", "object",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
errors.Trace(storage.AzureStorageServiceError{
|
||||
StatusCode: http.StatusNotFound,
|
||||
}), BucketNotFound{Bucket: "bucket"}, "bucket", "",
|
||||
}), minio.BucketNotFound{Bucket: "bucket"}, "bucket", "",
|
||||
},
|
||||
{
|
||||
traceError(storage.AzureStorageServiceError{
|
||||
errors.Trace(storage.AzureStorageServiceError{
|
||||
StatusCode: http.StatusBadRequest,
|
||||
}), BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
|
||||
}), minio.BucketNameInvalid{Bucket: "bucket."}, "bucket.", "",
|
||||
},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
err := azureToObjectError(testCase.actualErr, testCase.bucket, testCase.object)
|
||||
if err != nil {
|
||||
if err := azureToObjectError(testCase.actualErr, testCase.bucket, testCase.object); err != nil {
|
||||
if err.Error() != testCase.expectedErr.Error() {
|
||||
t.Errorf("Test %d: Expected error %s, got %s", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
@@ -190,32 +224,41 @@ func TestAzureParseBlockID(t *testing.T) {
|
||||
subPartNumber int
|
||||
uploadID string
|
||||
md5 string
|
||||
success bool
|
||||
}{
|
||||
{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ==", 1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e"},
|
||||
{"MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw==", 2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273"},
|
||||
// Invalid base64.
|
||||
{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ=", 0, 0, "", "", false},
|
||||
// Invalid number of tokens.
|
||||
{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNwo=", 0, 0, "", "", false},
|
||||
// Invalid encoded part ID.
|
||||
{"MDAwMGEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
|
||||
// Invalid sub part ID.
|
||||
{"MDAwMDEuQUEuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQo=", 0, 0, "", "", false},
|
||||
{"MDAwMDEuMDcuZjMyOGMzNWNhZDkzODEzNy5kNDFkOGNkOThmMDBiMjA0ZTk4MDA5OThlY2Y4NDI3ZQ==", 1, 7, "f328c35cad938137", "d41d8cd98f00b204e9800998ecf8427e", true},
|
||||
{"MDAwMDIuMTkuYWJjZGMzNWNhZDkzODEzNy5hN2ZiNmI3YjM2ZWU0ZWQ2NmI1NTQ2ZmFjNDY5MDI3Mw==", 2, 19, "abcdc35cad938137", "a7fb6b7b36ee4ed66b5546fac4690273", true},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
for i, test := range testCases {
|
||||
partID, subPartNumber, uploadID, md5, err := azureParseBlockID(test.blockID)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
if err != nil && test.success {
|
||||
t.Errorf("Test %d: Expected success but failed %s", i+1, err)
|
||||
}
|
||||
if partID != test.partID {
|
||||
t.Fatalf("%d not equal to %d", partID, test.partID)
|
||||
if err == nil && !test.success {
|
||||
t.Errorf("Test %d: Expected to fail but succeeeded insteadl", i+1)
|
||||
}
|
||||
if subPartNumber != test.subPartNumber {
|
||||
t.Fatalf("%d not equal to %d", subPartNumber, test.subPartNumber)
|
||||
if err == nil {
|
||||
if partID != test.partID {
|
||||
t.Errorf("Test %d: %d not equal to %d", i+1, partID, test.partID)
|
||||
}
|
||||
if subPartNumber != test.subPartNumber {
|
||||
t.Errorf("Test %d: %d not equal to %d", i+1, subPartNumber, test.subPartNumber)
|
||||
}
|
||||
if uploadID != test.uploadID {
|
||||
t.Errorf("Test %d: %s not equal to %s", i+1, uploadID, test.uploadID)
|
||||
}
|
||||
if md5 != test.md5 {
|
||||
t.Errorf("Test %d: %s not equal to %s", i+1, md5, test.md5)
|
||||
}
|
||||
}
|
||||
if uploadID != test.uploadID {
|
||||
t.Fatalf("%s not equal to %s", uploadID, test.uploadID)
|
||||
}
|
||||
if md5 != test.md5 {
|
||||
t.Fatalf("%s not equal to %s", md5, test.md5)
|
||||
}
|
||||
}
|
||||
|
||||
_, _, _, _, err := azureParseBlockID("junk")
|
||||
if err == nil {
|
||||
t.Fatal("Expected azureParseBlockID() to return error")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -279,32 +322,27 @@ func TestAnonErrToObjectErr(t *testing.T) {
|
||||
{"ObjectNotFound",
|
||||
http.StatusNotFound,
|
||||
[]string{"testBucket", "testObject"},
|
||||
ObjectNotFound{Bucket: "testBucket", Object: "testObject"},
|
||||
minio.ObjectNotFound{Bucket: "testBucket", Object: "testObject"},
|
||||
},
|
||||
{"BucketNotFound",
|
||||
http.StatusNotFound,
|
||||
[]string{"testBucket", ""},
|
||||
BucketNotFound{Bucket: "testBucket"},
|
||||
minio.BucketNotFound{Bucket: "testBucket"},
|
||||
},
|
||||
{"ObjectNameInvalid",
|
||||
http.StatusBadRequest,
|
||||
[]string{"testBucket", "testObject"},
|
||||
ObjectNameInvalid{Bucket: "testBucket", Object: "testObject"},
|
||||
minio.ObjectNameInvalid{Bucket: "testBucket", Object: "testObject"},
|
||||
},
|
||||
{"BucketNameInvalid",
|
||||
http.StatusBadRequest,
|
||||
[]string{"testBucket", ""},
|
||||
BucketNameInvalid{Bucket: "testBucket"},
|
||||
},
|
||||
{"UnexpectedError",
|
||||
http.StatusBadGateway,
|
||||
[]string{"testBucket", "testObject"},
|
||||
errUnexpected,
|
||||
minio.BucketNameInvalid{Bucket: "testBucket"},
|
||||
},
|
||||
}
|
||||
for _, test := range testCases {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if err := anonErrToObjectErr(test.statusCode, test.params...); !reflect.DeepEqual(err, test.wantErr) {
|
||||
if err := minio.AnonErrToObjectErr(test.statusCode, test.params...); !reflect.DeepEqual(err, test.wantErr) {
|
||||
t.Errorf("anonErrToObjectErr() error = %v, wantErr %v", err, test.wantErr)
|
||||
}
|
||||
})
|
||||
@@ -14,10 +14,9 @@
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
package b2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -25,6 +24,10 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/pkg/errors"
|
||||
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
// mkRange converts offset, size into Range header equivalent.
|
||||
@@ -33,18 +36,18 @@ func mkRange(offset, size int64) string {
|
||||
return ""
|
||||
}
|
||||
if size == 0 {
|
||||
return fmt.Sprintf("%s%d-", byteRangePrefix, offset)
|
||||
return fmt.Sprintf("bytes=%d-", offset)
|
||||
}
|
||||
return fmt.Sprintf("%s%d-%d", byteRangePrefix, offset, offset+size-1)
|
||||
return fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)
|
||||
}
|
||||
|
||||
// AnonGetObject - performs a plain http GET request on a public resource,
|
||||
// fails if the resource is not public.
|
||||
func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer) error {
|
||||
func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int64, length int64, writer io.Writer, etag string) error {
|
||||
uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
|
||||
req, err := http.NewRequest("GET", uri, nil)
|
||||
if err != nil {
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
return b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
rng := mkRange(startOffset, length)
|
||||
if rng != "" {
|
||||
@@ -52,14 +55,14 @@ func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int6
|
||||
}
|
||||
resp, err := l.anonClient.Do(req)
|
||||
if err != nil {
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
return b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
|
||||
return b2ToObjectError(errors.Trace(fmt.Errorf(resp.Status)), bucket, object)
|
||||
}
|
||||
_, err = io.Copy(writer, resp.Body)
|
||||
return b2ToObjectError(traceError(err), bucket, object)
|
||||
return b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
// Converts http Header into ObjectInfo. This function looks for all the
|
||||
@@ -70,16 +73,16 @@ func (l *b2Objects) AnonGetObject(bucket string, object string, startOffset int6
|
||||
// X-Bz-Info-<header>:<value> is converted to <header>:<value>
|
||||
// Content-Type is converted to ContentType.
|
||||
// X-Bz-Content-Sha1 is converted to ETag.
|
||||
func headerToObjectInfo(bucket, object string, header http.Header) (objInfo ObjectInfo, err error) {
|
||||
func headerToObjectInfo(bucket, object string, header http.Header) (objInfo minio.ObjectInfo, err error) {
|
||||
clen, err := strconv.ParseInt(header.Get("Content-Length"), 10, 64)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
// Converting upload timestamp in milliseconds to a time.Time value for ObjectInfo.ModTime.
|
||||
timeStamp, err := strconv.ParseInt(header.Get("X-Bz-Upload-Timestamp"), 10, 64)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
|
||||
// Populate user metadata by looking for all the X-Bz-Info-<name>
|
||||
@@ -91,18 +94,18 @@ func headerToObjectInfo(bucket, object string, header http.Header) (objInfo Obje
|
||||
var name string
|
||||
name, err = url.QueryUnescape(strings.TrimPrefix(key, "X-Bz-Info-"))
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
var val string
|
||||
val, err = url.QueryUnescape(header.Get(key))
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
userMetadata[name] = val
|
||||
}
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
return minio.ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
ContentType: header.Get("Content-Type"),
|
||||
@@ -115,19 +118,19 @@ func headerToObjectInfo(bucket, object string, header http.Header) (objInfo Obje
|
||||
|
||||
// AnonGetObjectInfo - performs a plain http HEAD request on a public resource,
|
||||
// fails if the resource is not public.
|
||||
func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo ObjectInfo, err error) {
|
||||
func (l *b2Objects) AnonGetObjectInfo(bucket string, object string) (objInfo minio.ObjectInfo, err error) {
|
||||
uri := fmt.Sprintf("%s/file/%s/%s", l.b2Client.DownloadURI, bucket, object)
|
||||
req, err := http.NewRequest("HEAD", uri, nil)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
resp, err := l.anonClient.Do(req)
|
||||
if err != nil {
|
||||
return objInfo, b2ToObjectError(traceError(err), bucket, object)
|
||||
return objInfo, b2ToObjectError(errors.Trace(err), bucket, object)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return objInfo, b2ToObjectError(traceError(errors.New(resp.Status)), bucket, object)
|
||||
return objInfo, b2ToObjectError(errors.Trace(fmt.Errorf(resp.Status)), bucket, object)
|
||||
}
|
||||
return headerToObjectInfo(bucket, object, resp.Header)
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user