mirror of
https://github.com/pgsty/minio.git
synced 2026-08-08 23:33:30 +03:00
Compare commits
93 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7090bcc8e0 | |||
| c222bde14b | |||
| 4e06a72632 | |||
| 16040bc544 | |||
| cc2d887e0e | |||
| c1b4b24236 | |||
| feaf8dfb9a | |||
| 628ef081d1 | |||
| 44dff36ff7 | |||
| b97d53b29c | |||
| 00af9881b0 | |||
| e09196d626 | |||
| 1a5775e2e8 | |||
| e2579b1f5a | |||
| 7824e19d20 | |||
| 317305d5f9 | |||
| e4e117faab | |||
| e8176fe978 | |||
| 828602d672 | |||
| d9224fbc65 | |||
| 51dad1d130 | |||
| 4593b146be | |||
| f21d650ed4 | |||
| a4f6705874 | |||
| b35b537e3f | |||
| 5c52d5ffc7 | |||
| f0808bb2e5 | |||
| a6dee21092 | |||
| 6f781c5e7a | |||
| f8ca859790 | |||
| b78521cd69 | |||
| 76e2713ffe | |||
| b5d291ea88 | |||
| eb9172eecb | |||
| 97a4c120e9 | |||
| 64bddf47d8 | |||
| 4ed45ce543 | |||
| ad511b0eb8 | |||
| cb0eaeaad8 | |||
| f3f0041ad0 | |||
| d0027c3c41 | |||
| cb7fc99368 | |||
| a4383051d9 | |||
| e7ae49f9c9 | |||
| 153d4be032 | |||
| dfd99b6d8f | |||
| c4b1d394d6 | |||
| c4131c2798 | |||
| c9d502e6fa | |||
| 677e80c0f8 | |||
| aa85af4d1a | |||
| ae731d232f | |||
| a317d220ed | |||
| 3e1221a01c | |||
| c1f6ca6697 | |||
| 36fc2f98ed | |||
| 556524c715 | |||
| 428f288379 | |||
| cde801282d | |||
| 6cf0008469 | |||
| 7b0330a98c | |||
| cc457f1798 | |||
| ca0d31b09a | |||
| 445a9bd827 | |||
| d8d25a308f | |||
| c19e6ce773 | |||
| d3c853a3be | |||
| 59d3639396 | |||
| 027e17468a | |||
| 45ea161f8d | |||
| 7b8a456f68 | |||
| b43906f6ee | |||
| 6a66f142d4 | |||
| 5982965839 | |||
| bfb92a27b7 | |||
| 8565cefe4e | |||
| 8cdf2106b0 | |||
| 35fafb837b | |||
| 274bbad5cb | |||
| 5c451d1690 | |||
| c987313431 | |||
| 2ecaab55a6 | |||
| 3e792ae2a2 | |||
| 6df6ac0f34 | |||
| 4cc500a041 | |||
| d8e28830cf | |||
| 3e16ec457a | |||
| e5d378931d | |||
| 6128304f6e | |||
| e63a10e505 | |||
| 5434088c51 | |||
| a773cf48d8 | |||
| 386dd56856 |
@@ -10,9 +10,10 @@
|
||||
## Types of changes
|
||||
- [ ] Bug fix (non-breaking change which fixes an issue)
|
||||
- [ ] New feature (non-breaking change which adds functionality)
|
||||
- [ ] Optimization (provides speedup with no functional changes)
|
||||
- [ ] Breaking change (fix or feature that would cause existing functionality to change)
|
||||
|
||||
## Checklist:
|
||||
- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here)
|
||||
- [ ] Documentation needed
|
||||
- [ ] Unit tests needed
|
||||
- [ ] Documentation updated
|
||||
- [ ] Unit tests added/updated
|
||||
|
||||
@@ -39,7 +39,7 @@ jobs:
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
sudo apt-get install devscripts shellcheck
|
||||
nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
|
||||
curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy
|
||||
curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-${nancy_version}-linux-amd64 && chmod +x nancy
|
||||
go list -m all | ./nancy sleuth
|
||||
make
|
||||
diff -au <(gofmt -s -d cmd) <(printf "")
|
||||
|
||||
@@ -15,6 +15,8 @@ FROM registry.access.redhat.com/ubi8/ubi-minimal:8.3
|
||||
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
|
||||
MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
|
||||
|
||||
@@ -17,6 +17,8 @@ ARG TARGETARCH
|
||||
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
|
||||
MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
|
||||
|
||||
@@ -12,6 +12,8 @@ COPY LICENSE /licenses/LICENSE
|
||||
ENV MINIO_UPDATE=off \
|
||||
MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
|
||||
MINIO_SSE_MASTER_KEY_FILE=sse_master_key
|
||||
|
||||
|
||||
+5
-3
@@ -5,16 +5,18 @@ ARG TARGETARCH
|
||||
LABEL name="MinIO" \
|
||||
vendor="MinIO Inc <dev@min.io>" \
|
||||
maintainer="MinIO Inc <dev@min.io>" \
|
||||
version="RELEASE.2020-11-25T22-36-25Z" \
|
||||
release="RELEASE.2020-11-25T22-36-25Z" \
|
||||
version="RELEASE.2021-01-08T21-18-21Z" \
|
||||
release="RELEASE.2021-01-08T21-18-21Z" \
|
||||
summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
|
||||
description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
|
||||
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
|
||||
MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
|
||||
|
||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
@@ -17,9 +17,9 @@ checks:
|
||||
getdeps:
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
|
||||
@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...)
|
||||
@which msgp 1>/dev/null || (echo "Installing msgp" && GO111MODULE=off go get github.com/tinylib/msgp)
|
||||
@which stringer 1>/dev/null || (echo "Installing stringer" && GO111MODULE=off go get golang.org/x/tools/cmd/stringer)
|
||||
@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1)
|
||||
@which msgp 1>/dev/null || (echo "Installing msgp" && go get github.com/tinylib/msgp@v1.1.3)
|
||||
@which stringer 1>/dev/null || (echo "Installing stringer" && go get golang.org/x/tools/cmd/stringer)
|
||||
|
||||
crosscompile:
|
||||
@(env bash $(PWD)/buildscripts/cross-compile.sh)
|
||||
|
||||
@@ -5,80 +5,190 @@
|
||||
|
||||
MinIO is a High Performance Object Storage released under Apache License v2.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads.
|
||||
|
||||
## Docker Container
|
||||
### Stable
|
||||
```
|
||||
docker run -p 9000:9000 \
|
||||
-e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
minio/minio server /data
|
||||
This README provides quickstart instructions on running MinIO on baremetal hardware, including Docker-based installations. For Kubernetes environments,
|
||||
use the [MinIO Kubernetes Operator](https://github.com/minio/operator/blob/master/README.md).
|
||||
|
||||
**Table of Contents**
|
||||
|
||||
- [Docker Installation](#docker-installation)
|
||||
- [macOS Installation](#macos)
|
||||
- [GNU/Linux Installation](#gnulinux)
|
||||
- [Windows Installation](#microsoft-windows)
|
||||
- [FreeBSD Installation](#freebsd)
|
||||
- [Source Installation](#install-from-source)
|
||||
- [Deployment Recommendations](#deployment-recommendations)
|
||||
- [Test Connectivity to MinIO](#test-minio-connectivity)
|
||||
- [Upgrade MinIO](#upgrading-minio)
|
||||
- [Explore Further](#explore-further)
|
||||
- [Contribute to MinIO](#contribute-to-minio-project)
|
||||
- [License](#license)
|
||||
|
||||
# Docker Installation
|
||||
|
||||
Use the following commands to run a standalone MinIO server on a Docker container.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
|
||||
for more complete documentation.
|
||||
|
||||
## Stable
|
||||
|
||||
Run the following command to run the latest stable image of MinIO on a Docker container using an ephemeral data volume:
|
||||
|
||||
```sh
|
||||
docker run -p 9000:9000 minio/minio server /data
|
||||
```
|
||||
|
||||
### Edge
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
|
||||
web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
|
||||
root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
|
||||
|
||||
|
||||
> NOTE: To deploy MinIO on Docker with persistent storage, you must map local persistent directories from the host OS to the container using the
|
||||
`docker -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the Docker container.
|
||||
|
||||
## Edge
|
||||
|
||||
Run the following command to run the bleeding-edge image of MinIO on a Docker container using an ephemeral data volume:
|
||||
|
||||
```
|
||||
docker run -p 9000:9000 \
|
||||
-e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
minio/minio:edge server /data
|
||||
docker run -p 9000:9000 minio/minio:edge server /data
|
||||
```
|
||||
|
||||
> NOTE: Docker will not display the default keys unless you start the container with the `-it`(interactive TTY) argument. Generally, it is not recommended to use default keys with containers. Please visit MinIO Docker quickstart guide for more information [here](https://docs.min.io/docs/minio-docker-quickstart-guide)
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
|
||||
web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
|
||||
root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
|
||||
|
||||
|
||||
> NOTE: To deploy MinIO on Docker with persistent storage, you must map local persistent directories from the host OS to the container using the
|
||||
`docker -v` option. For example, `-v /mnt/data:/data` maps the host OS drive at `/mnt/data` to `/data` on the Docker container.
|
||||
|
||||
# macOS
|
||||
|
||||
Use the following commands to run a standalone MinIO server on macOS.
|
||||
|
||||
Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
|
||||
for more complete documentation.
|
||||
|
||||
## Homebrew (recommended)
|
||||
|
||||
Run the following command to install the latest stable MinIO package using [Homebrew](https://brew.sh/). Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
|
||||
|
||||
## macOS
|
||||
### Homebrew (recommended)
|
||||
Install minio packages using [Homebrew](https://brew.sh/)
|
||||
```sh
|
||||
brew install minio/stable/minio
|
||||
minio server /data
|
||||
```
|
||||
|
||||
> NOTE: If you previously installed minio using `brew install minio` then it is recommended that you reinstall minio from `minio/stable/minio` official repo instead.
|
||||
|
||||
```sh
|
||||
brew uninstall minio
|
||||
brew install minio/stable/minio
|
||||
```
|
||||
|
||||
### Binary Download
|
||||
| Platform | Architecture | URL |
|
||||
| ---------- | -------- | ------ |
|
||||
| Apple macOS | 64-bit Intel | https://dl.min.io/server/minio/release/darwin-amd64/minio |
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
|
||||
web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
|
||||
root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
|
||||
|
||||
## Binary Download
|
||||
|
||||
Use the following command to download and run a standalone MinIO server on macOS. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
|
||||
|
||||
```sh
|
||||
chmod 755 minio
|
||||
wget https://dl.min.io/server/minio/release/darwin-amd64/minio
|
||||
chmod +x minio
|
||||
./minio server /data
|
||||
```
|
||||
|
||||
## GNU/Linux
|
||||
### Binary Download
|
||||
| Platform | Architecture | URL |
|
||||
| ---------- | -------- | ------ |
|
||||
| GNU/Linux | 64-bit Intel | https://dl.min.io/server/minio/release/linux-amd64/minio |
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
|
||||
web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
|
||||
root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
|
||||
|
||||
|
||||
# GNU/Linux
|
||||
|
||||
Use the following command to run a standalone MinIO server on Linux hosts running 64-bit Intel/AMD architectures. Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
|
||||
|
||||
```sh
|
||||
wget https://dl.min.io/server/minio/release/linux-amd64/minio
|
||||
chmod +x minio
|
||||
./minio server /data
|
||||
```
|
||||
|
||||
| Platform | Architecture | URL |
|
||||
| ---------- | -------- | ------ |
|
||||
| GNU/Linux | ppc64le | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
|
||||
Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
|
||||
|
||||
The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
|
||||
|
||||
| Architecture | URL |
|
||||
| -------- | ------ |
|
||||
| 64-bit Intel/AMD | https://dl.min.io/server/minio/release/linux-amd64/minio |
|
||||
| 64-bit ARM | https://dl.min.io/server/minio/release/linux-arm64/minio |
|
||||
| 64-bit PowerPC LE (ppc64le) | https://dl.min.io/server/minio/release/linux-ppc64le/minio |
|
||||
| IBM Z-Series (S390X) | https://dl.min.io/server/minio/release/linux-s390x/minio |
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
|
||||
web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
|
||||
root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
|
||||
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
|
||||
for more complete documentation.
|
||||
|
||||
# Microsoft Windows
|
||||
|
||||
To run MinIO on 64-bit Windows hosts, download the MinIO executable from the following URL:
|
||||
|
||||
```sh
|
||||
wget https://dl.min.io/server/minio/release/linux-ppc64le/minio
|
||||
chmod +x minio
|
||||
./minio server /data
|
||||
https://dl.min.io/server/minio/release/windows-amd64/minio.exe
|
||||
```
|
||||
|
||||
## Microsoft Windows
|
||||
### Binary Download
|
||||
| Platform | Architecture | URL |
|
||||
| ---------- | -------- | ------ |
|
||||
| Microsoft Windows | 64-bit | https://dl.min.io/server/minio/release/windows-amd64/minio.exe |
|
||||
Use the following command to run a standalone MinIO server on the Windows host. Replace ``D:\`` with the path to the drive or directory in which you want MinIO to store data. You must change the terminal or powershell directory to the location of the ``minio.exe`` executable, *or* add the path to that directory to the system ``$PATH``:
|
||||
|
||||
```sh
|
||||
minio.exe server D:\Photos
|
||||
minio.exe server D:\
|
||||
```
|
||||
|
||||
## FreeBSD
|
||||
### Port
|
||||
Install minio packages using [pkg](https://github.com/freebsd/pkg), MinIO doesn't officially build FreeBSD binaries but is maintained by FreeBSD upstream [here](https://www.freshports.org/www/minio).
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
|
||||
web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
|
||||
root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
|
||||
for more complete documentation.
|
||||
|
||||
# FreeBSD
|
||||
|
||||
MinIO does not provide an official FreeBSD binary. However, FreeBSD maintains an [upstream release](https://www.freshports.org/www/minio) using [pkg](https://github.com/freebsd/pkg):
|
||||
|
||||
```sh
|
||||
pkg install minio
|
||||
@@ -87,13 +197,32 @@ sysrc minio_disks=/home/user/Photos
|
||||
service minio start
|
||||
```
|
||||
|
||||
## Install from Source
|
||||
Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
|
||||
# Install from Source
|
||||
|
||||
Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
|
||||
|
||||
```sh
|
||||
GO111MODULE=on go get github.com/minio/minio
|
||||
```
|
||||
|
||||
The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Browser, an embedded
|
||||
web-based object browser built into MinIO Server. Point a web browser running on the host machine to http://127.0.0.1:9000 and log in with the
|
||||
root credentials. You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server.
|
||||
|
||||
You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool. See
|
||||
[Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. For application developers,
|
||||
see https://docs.min.io/docs/ and click **MINIO SDKS** in the navigation to view MinIO SDKs for supported languages.
|
||||
|
||||
|
||||
> NOTE: Standalone MinIO servers are best suited for early development and evaluation. Certain features such as versioning, object locking, and bucket replication
|
||||
require distributed deploying MinIO with Erasure Coding. For extended development and production, deploy MinIO with Erasure Coding enabled - specifically,
|
||||
with a *minimum* of 4 drives per MinIO server. See [MinIO Erasure Code Quickstart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide.html)
|
||||
for more complete documentation.
|
||||
|
||||
MinIO strongly recommends *against* using compiled-from-source MinIO servers for production environments.
|
||||
|
||||
# Deployment Recommendations
|
||||
|
||||
## Allow port access for Firewalls
|
||||
|
||||
By default MinIO uses the port 9000 to listen for incoming connections. If your platform blocks the port by default, you may need to enable access to the port.
|
||||
@@ -149,6 +278,13 @@ iptables -A INPUT -p tcp --dport 9000:9010 -j ACCEPT
|
||||
service iptables restart
|
||||
```
|
||||
|
||||
## Pre-existing data
|
||||
When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
|
||||
|
||||
The above statement is also valid for all gateway backends.
|
||||
|
||||
# Test MinIO Connectivity
|
||||
|
||||
## Test using MinIO Browser
|
||||
MinIO Server comes with an embedded web based object browser. Point your web browser to http://127.0.0.1:9000 to ensure your server has started successfully.
|
||||
|
||||
@@ -157,12 +293,7 @@ MinIO Server comes with an embedded web based object browser. Point your web bro
|
||||
## Test using MinIO Client `mc`
|
||||
`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. Follow the MinIO Client [Quickstart Guide](https://docs.min.io/docs/minio-client-quickstart-guide) for further instructions.
|
||||
|
||||
## Pre-existing data
|
||||
When deployed on a single drive, MinIO server lets clients access any pre-existing data in the data directory. For example, if MinIO is started with the command `minio server /mnt/data`, any pre-existing data in the `/mnt/data` directory would be accessible to the clients.
|
||||
|
||||
The above statement is also valid for all gateway backends.
|
||||
|
||||
## Upgrading MinIO
|
||||
# Upgrading MinIO
|
||||
MinIO server supports rolling upgrades, i.e. you can update one MinIO instance at a time in a distributed cluster. This allows upgrades with no downtime. Upgrades can be done manually by replacing the binary with the latest release and restarting all servers in a rolling fashion. However, we recommend all our users to use [`mc admin update`](https://docs.min.io/docs/minio-admin-complete-guide.html#update) from the client. This will update all the nodes in the cluster simultaneously and restart them, as shown in the following command from the MinIO client (mc):
|
||||
|
||||
```
|
||||
@@ -171,7 +302,7 @@ mc admin update <minio alias, e.g., myminio>
|
||||
|
||||
> NOTE: some releases might not allow rolling upgrades, this is always called out in the release notes and it is generally advised to read release notes before upgrading. In such a situation `mc admin update` is the recommended upgrading mechanism to upgrade all servers at once.
|
||||
|
||||
### Important things to remember during MinIO upgrades
|
||||
## Important things to remember during MinIO upgrades
|
||||
|
||||
- `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
|
||||
- `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
|
||||
@@ -181,7 +312,7 @@ mc admin update <minio alias, e.g., myminio>
|
||||
- If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
|
||||
- If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
|
||||
|
||||
## Explore Further
|
||||
# Explore Further
|
||||
- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
|
||||
- [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)
|
||||
- [Use `aws-cli` with MinIO Server](https://docs.min.io/docs/aws-cli-with-minio)
|
||||
@@ -189,8 +320,8 @@ mc admin update <minio alias, e.g., myminio>
|
||||
- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/docs/golang-client-quickstart-guide)
|
||||
- [The MinIO documentation website](https://docs.min.io)
|
||||
|
||||
## Contribute to MinIO Project
|
||||
# Contribute to MinIO Project
|
||||
Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
|
||||
|
||||
## License
|
||||
# License
|
||||
Use of MinIO is governed by the Apache 2.0 License found at [LICENSE](https://github.com/minio/minio/blob/master/LICENSE).
|
||||
|
||||
+4
-4
@@ -8,16 +8,16 @@ MinIO是一个非常轻量的服务,可以很简单的和其他应用的结合
|
||||
### 稳定版
|
||||
```
|
||||
docker run -p 9000:9000 \
|
||||
-e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
-e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
minio/minio server /data
|
||||
```
|
||||
|
||||
### 尝鲜版
|
||||
```
|
||||
docker run -p 9000:9000 \
|
||||
-e "MINIO_ACCESS_KEY=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_SECRET_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
-e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
|
||||
-e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
|
||||
minio/minio:edge server /data
|
||||
```
|
||||
|
||||
|
||||
@@ -138,9 +138,10 @@ describe("Uploads actions", () => {
|
||||
objects: { currentPrefix: "pre1/" }
|
||||
})
|
||||
store.dispatch(uploadsActions.uploadFile(file))
|
||||
const objectPath = encodeURIComponent("pre1/file1")
|
||||
expect(open).toHaveBeenCalledWith(
|
||||
"PUT",
|
||||
"https://localhost:8080/upload/test1/pre1/file1",
|
||||
"https://localhost:8080/upload/test1/" + objectPath,
|
||||
true
|
||||
)
|
||||
expect(send).toHaveBeenCalledWith(file)
|
||||
|
||||
@@ -94,7 +94,7 @@ export const uploadFile = file => {
|
||||
_filePath = _filePath.substring(1)
|
||||
}
|
||||
const filePath = _filePath
|
||||
const objectName = `${currentPrefix}${filePath}`
|
||||
const objectName = encodeURIComponent(`${currentPrefix}${filePath}`)
|
||||
const uploadUrl = `${
|
||||
window.location.origin
|
||||
}${minioBrowserPrefix}/upload/${currentBucket}/${objectName}`
|
||||
|
||||
+47
-45
File diff suppressed because one or more lines are too long
@@ -1,69 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# MinIO Cloud Storage, (C) 2019 MinIO, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set -e
|
||||
set -E
|
||||
set -o pipefail
|
||||
|
||||
function start_minio_server()
|
||||
{
|
||||
MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
|
||||
minio --quiet --json server /data --address 127.0.0.1:24242 > server.log 2>&1 &
|
||||
server_pid=$!
|
||||
sleep 10
|
||||
|
||||
echo "$server_pid"
|
||||
}
|
||||
|
||||
function start_minio_gateway_s3()
|
||||
{
|
||||
MINIO_ACCESS_KEY=minio MINIO_SECRET_KEY=minio123 \
|
||||
minio --quiet --json gateway s3 http://127.0.0.1:24242 \
|
||||
--address 127.0.0.1:24240 > gateway.log 2>&1 &
|
||||
gw_pid=$!
|
||||
sleep 10
|
||||
|
||||
echo "$gw_pid"
|
||||
}
|
||||
|
||||
function main()
|
||||
{
|
||||
sr_pid="$(start_minio_server)"
|
||||
gw_pid="$(start_minio_gateway_s3)"
|
||||
|
||||
SERVER_ENDPOINT=127.0.0.1:24240 ENABLE_HTTPS=0 ACCESS_KEY=minio \
|
||||
SECRET_KEY=minio123 MINT_MODE="full" /mint/entrypoint.sh \
|
||||
aws-sdk-go aws-sdk-java aws-sdk-php aws-sdk-ruby awscli \
|
||||
healthcheck mc minio-dotnet minio-js \
|
||||
minio-py s3cmd s3select security
|
||||
rv=$?
|
||||
|
||||
kill "$sr_pid"
|
||||
kill "$gw_pid"
|
||||
sleep 3
|
||||
|
||||
if [ "$rv" -ne 0 ]; then
|
||||
echo "=========== Gateway ==========="
|
||||
cat "gateway.log"
|
||||
echo "=========== Server ==========="
|
||||
cat "server.log"
|
||||
fi
|
||||
|
||||
rm -f gateway.log server.log
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -3,5 +3,5 @@
|
||||
set -e
|
||||
|
||||
for d in $(go list ./... | grep -v browser); do
|
||||
CGO_ENABLED=1 go test -v -race --timeout 50m "$d"
|
||||
CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
|
||||
done
|
||||
|
||||
@@ -63,8 +63,8 @@ function start_minio_erasure_sets()
|
||||
|
||||
function start_minio_pool_erasure_sets()
|
||||
{
|
||||
export MINIO_ACCESS_KEY=$ACCESS_KEY
|
||||
export MINIO_SECRET_KEY=$SECRET_KEY
|
||||
export MINIO_ROOT_USER=$ACCESS_KEY
|
||||
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
||||
export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
|
||||
"${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
|
||||
"${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
|
||||
@@ -74,8 +74,8 @@ function start_minio_pool_erasure_sets()
|
||||
|
||||
function start_minio_pool_erasure_sets_ipv6()
|
||||
{
|
||||
export MINIO_ACCESS_KEY=$ACCESS_KEY
|
||||
export MINIO_SECRET_KEY=$SECRET_KEY
|
||||
export MINIO_ROOT_USER=$ACCESS_KEY
|
||||
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
||||
export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
|
||||
"${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
|
||||
"${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
|
||||
@@ -85,8 +85,8 @@ function start_minio_pool_erasure_sets_ipv6()
|
||||
|
||||
function start_minio_dist_erasure()
|
||||
{
|
||||
export MINIO_ACCESS_KEY=$ACCESS_KEY
|
||||
export MINIO_SECRET_KEY=$SECRET_KEY
|
||||
export MINIO_ROOT_USER=$ACCESS_KEY
|
||||
export MINIO_ROOT_PASSWORD=$SECRET_KEY
|
||||
export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
|
||||
for i in $(seq 0 3); do
|
||||
"${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
|
||||
|
||||
@@ -29,8 +29,8 @@ MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||
MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
|
||||
|
||||
function start_minio_3_node() {
|
||||
export MINIO_ACCESS_KEY=minio
|
||||
export MINIO_SECRET_KEY=minio123
|
||||
export MINIO_ROOT_USER=minio
|
||||
export MINIO_ROOT_PASSWORD=minio123
|
||||
export MINIO_ERASURE_SET_DRIVE_COUNT=6
|
||||
|
||||
start_port=$(shuf -i 10000-65000 -n 1)
|
||||
|
||||
@@ -23,9 +23,7 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
iampolicy "github.com/minio/minio/pkg/iam/policy"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
@@ -54,12 +52,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
// Turn off quota commands if data usage info is unavailable.
|
||||
if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminBucketQuotaDisabled), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -194,7 +186,6 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = globalBucketMetadataSys.Update(bucket, bucketTargetsFile, tgtBytes); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
|
||||
@@ -49,7 +49,7 @@ func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *htt
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
cred, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
|
||||
cred, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ConfigUpdateAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return cred, nil
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"path"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
@@ -42,7 +43,7 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
|
||||
cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return nil, cred
|
||||
@@ -358,7 +359,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
defer logger.AuditLog(w, r, "AddUser", mustGetClaimsFromToken(r))
|
||||
|
||||
vars := mux.Vars(r)
|
||||
accessKey := vars["accessKey"]
|
||||
accessKey := path.Clean(vars["accessKey"])
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
@@ -373,23 +374,29 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if cred.IsTemp() || cred.IsServiceAccount() {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccountNotEligible), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// Not allowed to add a user with same access key as root credential
|
||||
if owner && accessKey == cred.AccessKey {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
|
||||
// Incoming access key matches parent user then we should
|
||||
// reject password change requests.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
implicitPerm := accessKey == cred.AccessKey
|
||||
if !implicitPerm {
|
||||
parentUser := cred.ParentUser
|
||||
if parentUser == "" {
|
||||
parentUser = cred.AccessKey
|
||||
}
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
AccountName: parentUser,
|
||||
Action: iampolicy.CreateUserAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
|
||||
ConditionValues: getConditionValues(r, "", parentUser, claims),
|
||||
IsOwner: owner,
|
||||
Claims: claims,
|
||||
}) {
|
||||
|
||||
+71
-62
@@ -292,7 +292,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
|
||||
// ignores any errors here.
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx, false)
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx)
|
||||
|
||||
// Collect any disk healing.
|
||||
healing, _ := getAggregatedBackgroundHealState(ctx)
|
||||
@@ -461,8 +461,15 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
|
||||
|
||||
defer logger.AuditLog(w, r, "StartProfiling", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
|
||||
if objectAPI == nil {
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -559,8 +566,15 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
defer logger.AuditLog(w, r, "DownloadProfiling", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ProfilingAdminAction)
|
||||
if objectAPI == nil {
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -900,7 +914,7 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
|
||||
}
|
||||
|
||||
// Validate request signature.
|
||||
cred, adminAPIErr = checkAdminRequestAuthType(ctx, r, action, "")
|
||||
cred, adminAPIErr = checkAdminRequestAuth(ctx, r, action, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return nil, cred
|
||||
@@ -1032,7 +1046,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
trcErr := r.URL.Query().Get("err") == "true"
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.TraceAdminAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -1398,7 +1412,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
}()
|
||||
|
||||
ticker := time.NewTicker(30 * time.Second)
|
||||
ticker := time.NewTicker(5 * time.Second)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
@@ -1431,7 +1445,7 @@ func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http
|
||||
defer logger.AuditLog(w, r, "BandwidthMonitor", mustGetClaimsFromToken(r))
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.BandwidthMonitorAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -1483,22 +1497,13 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
defer logger.AuditLog(w, r, "ServerInfo", mustGetClaimsFromToken(r))
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
buckets := madmin.Buckets{}
|
||||
objects := madmin.Objects{}
|
||||
usage := madmin.Usage{}
|
||||
|
||||
dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
|
||||
if err == nil {
|
||||
buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
|
||||
objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
|
||||
usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
|
||||
}
|
||||
|
||||
vault := fetchVaultStatus()
|
||||
|
||||
ldap := madmin.LDAP{}
|
||||
@@ -1520,31 +1525,54 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
// Get the notification target info
|
||||
notifyTarget := fetchLambdaInfo()
|
||||
|
||||
// Fetching the Storage information, ignore any errors.
|
||||
storageInfo, _ := objectAPI.StorageInfo(ctx, false)
|
||||
|
||||
var backend interface{}
|
||||
if storageInfo.Backend.Type == BackendType(madmin.Erasure) {
|
||||
backend = madmin.ErasureBackend{
|
||||
Type: madmin.ErasureType,
|
||||
OnlineDisks: storageInfo.Backend.OnlineDisks.Sum(),
|
||||
OfflineDisks: storageInfo.Backend.OfflineDisks.Sum(),
|
||||
StandardSCData: storageInfo.Backend.StandardSCData,
|
||||
StandardSCParity: storageInfo.Backend.StandardSCParity,
|
||||
RRSCData: storageInfo.Backend.RRSCData,
|
||||
RRSCParity: storageInfo.Backend.RRSCParity,
|
||||
}
|
||||
} else {
|
||||
backend = madmin.FSBackend{
|
||||
Type: madmin.FsType,
|
||||
}
|
||||
}
|
||||
|
||||
mode := "online"
|
||||
server := getLocalServerProperty(globalEndpoints, r)
|
||||
servers := globalNotificationSys.ServerInfo()
|
||||
servers = append(servers, server)
|
||||
|
||||
var backend interface{}
|
||||
mode := madmin.ObjectLayerInitializing
|
||||
|
||||
buckets := madmin.Buckets{}
|
||||
objects := madmin.Objects{}
|
||||
usage := madmin.Usage{}
|
||||
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI != nil {
|
||||
mode = madmin.ObjectLayerOnline
|
||||
// Load data usage
|
||||
dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
|
||||
if err == nil {
|
||||
buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
|
||||
objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
|
||||
usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
|
||||
}
|
||||
|
||||
// Fetching the backend information
|
||||
backendInfo := objectAPI.BackendInfo()
|
||||
if backendInfo.Type == BackendType(madmin.Erasure) {
|
||||
// Calculate the number of online/offline disks of all nodes
|
||||
var allDisks []madmin.Disk
|
||||
for _, s := range servers {
|
||||
allDisks = append(allDisks, s.Disks...)
|
||||
}
|
||||
onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
|
||||
|
||||
backend = madmin.ErasureBackend{
|
||||
Type: madmin.ErasureType,
|
||||
OnlineDisks: onlineDisks.Sum(),
|
||||
OfflineDisks: offlineDisks.Sum(),
|
||||
StandardSCData: backendInfo.StandardSCData,
|
||||
StandardSCParity: backendInfo.StandardSCParity,
|
||||
RRSCData: backendInfo.RRSCData,
|
||||
RRSCParity: backendInfo.RRSCParity,
|
||||
}
|
||||
} else {
|
||||
backend = madmin.FSBackend{
|
||||
Type: madmin.FsType,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
domain := globalDomainNames
|
||||
services := madmin.Services{
|
||||
Vault: vault,
|
||||
@@ -1554,25 +1582,6 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
Notifications: notifyTarget,
|
||||
}
|
||||
|
||||
// find all disks which belong to each respective endpoints
|
||||
for i := range servers {
|
||||
for _, disk := range storageInfo.Disks {
|
||||
if strings.Contains(disk.Endpoint, servers[i].Endpoint) {
|
||||
servers[i].Disks = append(servers[i].Disks, disk)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// add all the disks local to this server.
|
||||
for _, disk := range storageInfo.Disks {
|
||||
if disk.DrivePath == "" && disk.Endpoint == "" {
|
||||
continue
|
||||
}
|
||||
if disk.Endpoint == disk.DrivePath {
|
||||
servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
|
||||
}
|
||||
}
|
||||
|
||||
infoMsg := madmin.InfoMessage{
|
||||
Mode: mode,
|
||||
Domain: domain,
|
||||
|
||||
@@ -364,7 +364,7 @@ func TestExtractHealInitParams(t *testing.T) {
|
||||
// Test all combinations!
|
||||
for pIdx, parms := range qParmsArr {
|
||||
for vIdx, vars := range varsArr {
|
||||
_, err := extractHealInitParams(vars, parms, bytes.NewBuffer([]byte(body)))
|
||||
_, err := extractHealInitParams(vars, parms, bytes.NewReader([]byte(body)))
|
||||
isErrCase := false
|
||||
if pIdx < 4 || vIdx < 1 {
|
||||
isErrCase = true
|
||||
|
||||
@@ -677,10 +677,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
|
||||
}
|
||||
if source.opts != nil {
|
||||
task.opts = *source.opts
|
||||
} else {
|
||||
if opts.Bitrot {
|
||||
task.opts.ScanMode = madmin.HealDeepScan
|
||||
}
|
||||
}
|
||||
if opts.Bitrot {
|
||||
task.opts.ScanMode = madmin.HealDeepScan
|
||||
}
|
||||
|
||||
// Wait and proceed if there are active requests
|
||||
|
||||
+17
-21
@@ -20,8 +20,6 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/gorilla/mux"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
)
|
||||
|
||||
@@ -172,27 +170,25 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
|
||||
}
|
||||
|
||||
if globalIsDistErasure || globalIsErasure {
|
||||
// Quota operations
|
||||
if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
|
||||
// GetBucketQuotaConfig
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
|
||||
// PutBucketQuotaConfig
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
|
||||
// GetBucketQuotaConfig
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
|
||||
// PutBucketQuotaConfig
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
|
||||
|
||||
// Bucket replication operations
|
||||
// GetBucketTargetHandler
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
|
||||
// SetRemoteTargetHandler
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
|
||||
// RemoveRemoteTargetHandler
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
|
||||
}
|
||||
// Bucket replication operations
|
||||
// GetBucketTargetHandler
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
|
||||
// SetRemoteTargetHandler
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
|
||||
// RemoveRemoteTargetHandler
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
|
||||
httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
|
||||
}
|
||||
|
||||
// -- Top APIs --
|
||||
// Top locks
|
||||
if globalIsDistErasure {
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
// getLocalServerProperty - returns madmin.ServerProperties for only the
|
||||
// local endpoints from given list of endpoints
|
||||
func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
|
||||
var localEndpoints Endpoints
|
||||
addr := r.Host
|
||||
if globalIsDistErasure {
|
||||
addr = GetLocalPeer(endpointServerPools)
|
||||
@@ -39,11 +40,12 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
if endpoint.IsLocal {
|
||||
// Only proceed for local endpoints
|
||||
network[nodeName] = "online"
|
||||
localEndpoints = append(localEndpoints, endpoint)
|
||||
continue
|
||||
}
|
||||
_, present := network[nodeName]
|
||||
if !present {
|
||||
if err := IsServerResolvable(endpoint); err == nil {
|
||||
if err := isServerResolvable(endpoint); err == nil {
|
||||
network[nodeName] = "online"
|
||||
} else {
|
||||
network[nodeName] = "offline"
|
||||
@@ -52,6 +54,11 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
localDisks, _ := initStorageDisksWithErrors(localEndpoints)
|
||||
defer closeStorageDisks(localDisks)
|
||||
|
||||
storageInfo, _ := getStorageInfo(localDisks, localEndpoints.GetAllStrings())
|
||||
|
||||
return madmin.ServerProperties{
|
||||
State: "ok",
|
||||
Endpoint: addr,
|
||||
@@ -59,5 +66,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
Version: Version,
|
||||
CommitID: CommitID,
|
||||
Network: network,
|
||||
Disks: storageInfo.Disks,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -120,7 +120,6 @@ const (
|
||||
ErrReplicationSourceNotVersionedError
|
||||
ErrReplicationNeedsVersioningError
|
||||
ErrReplicationBucketNeedsVersioningError
|
||||
ErrBucketReplicationDisabledError
|
||||
ErrObjectRestoreAlreadyInProgress
|
||||
ErrNoSuchKey
|
||||
ErrNoSuchUpload
|
||||
@@ -263,7 +262,6 @@ const (
|
||||
// Bucket Quota error codes
|
||||
ErrAdminBucketQuotaExceeded
|
||||
ErrAdminNoSuchQuotaConfiguration
|
||||
ErrAdminBucketQuotaDisabled
|
||||
|
||||
ErrHealNotImplemented
|
||||
ErrHealNoSuchProcess
|
||||
@@ -889,11 +887,6 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Versioning must be 'Enabled' on the bucket to add a replication target",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrBucketReplicationDisabledError: {
|
||||
Code: "XMinioAdminBucketReplicationDisabled",
|
||||
Description: "Replication specified but disk usage crawl is disabled on MinIO server",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrNoSuchObjectLockConfiguration: {
|
||||
Code: "NoSuchObjectLockConfiguration",
|
||||
Description: "The specified object does not have a ObjectLock configuration",
|
||||
@@ -1215,11 +1208,6 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The quota configuration does not exist",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrAdminBucketQuotaDisabled: {
|
||||
Code: "XMinioAdminBucketQuotaDisabled",
|
||||
Description: "Quota specified but disk usage crawl is disabled on MinIO server",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrInsecureClientRequest: {
|
||||
Code: "XMinioInsecureClientRequest",
|
||||
Description: "Cannot respond to plain-text request from TLS-encrypted server",
|
||||
|
||||
+1
-1
@@ -388,7 +388,7 @@ func getObjectLocation(r *http.Request, domains []string, bucket, object string)
|
||||
}
|
||||
proto := handlers.GetSourceScheme(r)
|
||||
if proto == "" {
|
||||
proto = getURLScheme(globalIsSSL)
|
||||
proto = getURLScheme(globalIsTLS)
|
||||
}
|
||||
u := &url.URL{
|
||||
Host: r.Host,
|
||||
|
||||
+87
-74
@@ -27,23 +27,35 @@ import (
|
||||
)
|
||||
|
||||
func newHTTPServerFn() *xhttp.Server {
|
||||
globalObjLayerMutex.Lock()
|
||||
defer globalObjLayerMutex.Unlock()
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
return globalHTTPServer
|
||||
}
|
||||
|
||||
func newObjectLayerFn() ObjectLayer {
|
||||
func setHTTPServer(h *xhttp.Server) {
|
||||
globalObjLayerMutex.Lock()
|
||||
defer globalObjLayerMutex.Unlock()
|
||||
globalHTTPServer = h
|
||||
globalObjLayerMutex.Unlock()
|
||||
}
|
||||
|
||||
func newObjectLayerFn() ObjectLayer {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
return globalObjectAPI
|
||||
}
|
||||
|
||||
func newCachedObjectLayerFn() CacheObjectLayer {
|
||||
globalObjLayerMutex.Lock()
|
||||
defer globalObjLayerMutex.Unlock()
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
return globalCacheObjectAPI
|
||||
}
|
||||
|
||||
func setCacheObjectLayer(c CacheObjectLayer) {
|
||||
globalObjLayerMutex.Lock()
|
||||
globalCacheObjectAPI = c
|
||||
globalObjLayerMutex.Unlock()
|
||||
}
|
||||
|
||||
func setObjectLayer(o ObjectLayer) {
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = o
|
||||
@@ -108,224 +120,225 @@ func registerAPIRouter(router *mux.Router) {
|
||||
// Object operations
|
||||
// HeadObject
|
||||
bucket.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("headobject", httpTraceAll(api.HeadObjectHandler))))
|
||||
collectAPIStats("headobject", maxClients(httpTraceAll(api.HeadObjectHandler))))
|
||||
// CopyObjectPart
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").
|
||||
HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
|
||||
HandlerFunc(maxClients(collectAPIStats("copyobjectpart", httpTraceAll(api.CopyObjectPartHandler)))).
|
||||
HandlerFunc(collectAPIStats("copyobjectpart", maxClients(httpTraceAll(api.CopyObjectPartHandler)))).
|
||||
Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// PutObjectPart
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("putobjectpart", httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
collectAPIStats("putobjectpart", maxClients(httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
|
||||
// ListObjectParts
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectparts", httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
|
||||
collectAPIStats("listobjectparts", maxClients(httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
|
||||
// CompleteMultipartUpload
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("completemutipartupload", httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
|
||||
collectAPIStats("completemutipartupload", maxClients(httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
|
||||
// NewMultipartUpload
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("newmultipartupload", httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
|
||||
collectAPIStats("newmultipartupload", maxClients(httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
|
||||
// AbortMultipartUpload
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("abortmultipartupload", httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
|
||||
collectAPIStats("abortmultipartupload", maxClients(httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
|
||||
// GetObjectACL - this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("getobjectacl", httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
|
||||
collectAPIStats("getobjectacl", maxClients(httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
|
||||
// PutObjectACL - this is a dummy call.
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("putobjectacl", httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
|
||||
collectAPIStats("putobjectacl", maxClients(httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
|
||||
// GetObjectTagging
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("getobjecttagging", httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
|
||||
collectAPIStats("getobjecttagging", maxClients(httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
|
||||
// PutObjectTagging
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("putobjecttagging", httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
|
||||
collectAPIStats("putobjecttagging", maxClients(httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
|
||||
// DeleteObjectTagging
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("deleteobjecttagging", httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
|
||||
collectAPIStats("deleteobjecttagging", maxClients(httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
|
||||
// SelectObjectContent
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("selectobjectcontent", httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
|
||||
collectAPIStats("selectobjectcontent", maxClients(httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
|
||||
// GetObjectRetention
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("getobjectretention", httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
|
||||
collectAPIStats("getobjectretention", maxClients(httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
|
||||
// GetObjectLegalHold
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("getobjectlegalhold", httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
|
||||
collectAPIStats("getobjectlegalhold", maxClients(httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
|
||||
// GetObject
|
||||
bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("getobject", httpTraceHdrs(api.GetObjectHandler))))
|
||||
collectAPIStats("getobject", maxClients(httpTraceHdrs(api.GetObjectHandler))))
|
||||
// CopyObject
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
|
||||
HandlerFunc(maxClients(collectAPIStats("copyobject", httpTraceAll(api.CopyObjectHandler))))
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
|
||||
collectAPIStats("copyobject", maxClients(httpTraceAll(api.CopyObjectHandler))))
|
||||
// PutObjectRetention
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("putobjectretention", httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
|
||||
collectAPIStats("putobjectretention", maxClients(httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
|
||||
// PutObjectLegalHold
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("putobjectlegalhold", httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
|
||||
collectAPIStats("putobjectlegalhold", maxClients(httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
|
||||
|
||||
// PutObject
|
||||
bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("putobject", httpTraceHdrs(api.PutObjectHandler))))
|
||||
collectAPIStats("putobject", maxClients(httpTraceHdrs(api.PutObjectHandler))))
|
||||
// DeleteObject
|
||||
bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("deleteobject", httpTraceAll(api.DeleteObjectHandler))))
|
||||
collectAPIStats("deleteobject", maxClients(httpTraceAll(api.DeleteObjectHandler))))
|
||||
|
||||
/// Bucket operations
|
||||
// GetBucketLocation
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketlocation", httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
|
||||
collectAPIStats("getbucketlocation", maxClients(httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
|
||||
// GetBucketPolicy
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketpolicy", httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
|
||||
collectAPIStats("getbucketpolicy", maxClients(httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
|
||||
// GetBucketLifecycle
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
// GetBucketEncryption
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketencryption", httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
|
||||
collectAPIStats("getbucketencryption", maxClients(httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
|
||||
// GetBucketObjectLockConfig
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketobjectlockconfiguration", httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
|
||||
collectAPIStats("getbucketobjectlockconfiguration", maxClients(httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
|
||||
// GetBucketReplicationConfig
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketreplicationconfiguration", httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
|
||||
collectAPIStats("getbucketreplicationconfiguration", maxClients(httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
|
||||
|
||||
// GetBucketVersioning
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketversioning", httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
|
||||
collectAPIStats("getbucketversioning", maxClients(httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
|
||||
// GetBucketNotification
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketnotification", httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
|
||||
collectAPIStats("getbucketnotification", maxClients(httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
|
||||
// ListenNotification
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
|
||||
|
||||
// Dummy Bucket Calls
|
||||
// GetBucketACL -- this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketacl", httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
|
||||
collectAPIStats("getbucketacl", maxClients(httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
|
||||
// PutBucketACL -- this is a dummy call.
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketacl", httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
|
||||
collectAPIStats("putbucketacl", maxClients(httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
|
||||
// GetBucketCors - this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketcors", httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
|
||||
collectAPIStats("getbucketcors", maxClients(httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
|
||||
// GetBucketWebsiteHandler - this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketwebsite", httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
|
||||
collectAPIStats("getbucketwebsite", maxClients(httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
|
||||
// GetBucketAccelerateHandler - this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketaccelerate", httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
|
||||
collectAPIStats("getbucketaccelerate", maxClients(httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
|
||||
// GetBucketRequestPaymentHandler - this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketrequestpayment", httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
|
||||
collectAPIStats("getbucketrequestpayment", maxClients(httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
|
||||
// GetBucketLoggingHandler - this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketlogging", httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
|
||||
collectAPIStats("getbucketlogging", maxClients(httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
|
||||
// GetBucketLifecycleHandler - this is a dummy call.
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbucketlifecycle", httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
// GetBucketTaggingHandler
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("getbuckettagging", httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
|
||||
collectAPIStats("getbuckettagging", maxClients(httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
|
||||
//DeleteBucketWebsiteHandler
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebucketwebsite", httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
|
||||
collectAPIStats("deletebucketwebsite", maxClients(httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
|
||||
// DeleteBucketTaggingHandler
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebuckettagging", httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
|
||||
collectAPIStats("deletebuckettagging", maxClients(httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
|
||||
|
||||
// ListMultipartUploads
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listmultipartuploads", httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
|
||||
collectAPIStats("listmultipartuploads", maxClients(httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
|
||||
// ListObjectsV2M
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectsv2M", httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
|
||||
collectAPIStats("listobjectsv2M", maxClients(httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
|
||||
// ListObjectsV2
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectsv2", httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
|
||||
collectAPIStats("listobjectsv2", maxClients(httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
|
||||
// ListObjectVersions
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectversions", httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
|
||||
collectAPIStats("listobjectversions", maxClients(httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
|
||||
// ListObjectsV1 (Legacy)
|
||||
bucket.Methods(http.MethodGet).HandlerFunc(
|
||||
maxClients(collectAPIStats("listobjectsv1", httpTraceAll(api.ListObjectsV1Handler))))
|
||||
collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler))))
|
||||
// PutBucketLifecycle
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketlifecycle", httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
collectAPIStats("putbucketlifecycle", maxClients(httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
// PutBucketReplicationConfig
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketreplicationconfiguration", httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
|
||||
collectAPIStats("putbucketreplicationconfiguration", maxClients(httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
|
||||
// GetObjectRetention
|
||||
|
||||
// PutBucketEncryption
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketencryption", httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
|
||||
collectAPIStats("putbucketencryption", maxClients(httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
|
||||
|
||||
// PutBucketPolicy
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketpolicy", httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
|
||||
collectAPIStats("putbucketpolicy", maxClients(httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
|
||||
|
||||
// PutBucketObjectLockConfig
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketobjectlockconfig", httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
|
||||
collectAPIStats("putbucketobjectlockconfig", maxClients(httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
|
||||
// PutBucketTaggingHandler
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbuckettagging", httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
|
||||
collectAPIStats("putbuckettagging", maxClients(httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
|
||||
// PutBucketVersioning
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketversioning", httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
|
||||
collectAPIStats("putbucketversioning", maxClients(httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
|
||||
// PutBucketNotification
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucketnotification", httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
|
||||
collectAPIStats("putbucketnotification", maxClients(httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
|
||||
// PutBucket
|
||||
bucket.Methods(http.MethodPut).HandlerFunc(
|
||||
maxClients(collectAPIStats("putbucket", httpTraceAll(api.PutBucketHandler))))
|
||||
collectAPIStats("putbucket", maxClients(httpTraceAll(api.PutBucketHandler))))
|
||||
// HeadBucket
|
||||
bucket.Methods(http.MethodHead).HandlerFunc(
|
||||
maxClients(collectAPIStats("headbucket", httpTraceAll(api.HeadBucketHandler))))
|
||||
collectAPIStats("headbucket", maxClients(httpTraceAll(api.HeadBucketHandler))))
|
||||
// PostPolicy
|
||||
bucket.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
|
||||
maxClients(collectAPIStats("postpolicybucket", httpTraceHdrs(api.PostPolicyBucketHandler))))
|
||||
collectAPIStats("postpolicybucket", maxClients(httpTraceHdrs(api.PostPolicyBucketHandler))))
|
||||
// DeleteMultipleObjects
|
||||
bucket.Methods(http.MethodPost).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletemultipleobjects", httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
|
||||
collectAPIStats("deletemultipleobjects", maxClients(httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
|
||||
// DeleteBucketPolicy
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebucketpolicy", httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
|
||||
collectAPIStats("deletebucketpolicy", maxClients(httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
|
||||
// DeleteBucketReplication
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebucketreplicationconfiguration", httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
|
||||
collectAPIStats("deletebucketreplicationconfiguration", maxClients(httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
|
||||
// DeleteBucketLifecycle
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebucketlifecycle", httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
collectAPIStats("deletebucketlifecycle", maxClients(httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
|
||||
// DeleteBucketEncryption
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebucketencryption", httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
|
||||
collectAPIStats("deletebucketencryption", maxClients(httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
|
||||
// DeleteBucket
|
||||
bucket.Methods(http.MethodDelete).HandlerFunc(
|
||||
maxClients(collectAPIStats("deletebucket", httpTraceAll(api.DeleteBucketHandler))))
|
||||
collectAPIStats("deletebucket", maxClients(httpTraceAll(api.DeleteBucketHandler))))
|
||||
// PostRestoreObject
|
||||
bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
|
||||
maxClients(collectAPIStats("restoreobject", httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
|
||||
collectAPIStats("restoreobject", maxClients(httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
|
||||
}
|
||||
|
||||
/// Root operation
|
||||
|
||||
// ListenNotification
|
||||
apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
|
||||
collectAPIStats("listennotification", httpTraceAll(api.ListenNotificationHandler))).Queries("events", "{events:.*}")
|
||||
collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
|
||||
|
||||
// ListBuckets
|
||||
apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
|
||||
maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
|
||||
collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
|
||||
|
||||
// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
|
||||
// than failing with UnknownAPIRequest we simply handle it for now.
|
||||
apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
|
||||
maxClients(collectAPIStats("listbuckets", httpTraceAll(api.ListBucketsHandler))))
|
||||
collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
|
||||
|
||||
// If none of the routes match add default error handler routes
|
||||
apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
|
||||
|
||||
+25
-31
@@ -151,9 +151,10 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
|
||||
return cred, claims, owner, ErrNone
|
||||
}
|
||||
|
||||
// checkAdminRequestAuthType checks whether the request is a valid signature V2 or V4 request.
|
||||
// It does not accept presigned or JWT or anonymous requests.
|
||||
func checkAdminRequestAuthType(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
|
||||
// checkAdminRequestAuth checks for authentication and authorization for the incoming
|
||||
// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
|
||||
// are automatically rejected.
|
||||
func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
|
||||
cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
|
||||
if s3Err != ErrNone {
|
||||
return cred, s3Err
|
||||
@@ -467,16 +468,6 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
// authHandler - handles all the incoming authorization headers and validates them if possible.
|
||||
type authHandler struct {
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
// setAuthHandler to validate authorization header for the incoming request.
|
||||
func setAuthHandler(h http.Handler) http.Handler {
|
||||
return authHandler{h}
|
||||
}
|
||||
|
||||
// List of all support S3 auth types.
|
||||
var supportedS3AuthTypes = map[authType]struct{}{
|
||||
authTypeAnonymous: {},
|
||||
@@ -494,26 +485,29 @@ func isSupportedS3AuthType(aType authType) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
// handler for validating incoming authorization headers.
|
||||
func (a authHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
aType := getRequestAuthType(r)
|
||||
if isSupportedS3AuthType(aType) {
|
||||
// Let top level caller validate for anonymous and known signed requests.
|
||||
a.handler.ServeHTTP(w, r)
|
||||
return
|
||||
} else if aType == authTypeJWT {
|
||||
// Validate Authorization header if its valid for JWT request.
|
||||
if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
// setAuthHandler to validate authorization header for the incoming request.
|
||||
func setAuthHandler(h http.Handler) http.Handler {
|
||||
// handler for validating incoming authorization headers.
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
aType := getRequestAuthType(r)
|
||||
if isSupportedS3AuthType(aType) {
|
||||
// Let top level caller validate for anonymous and known signed requests.
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
} else if aType == authTypeJWT {
|
||||
// Validate Authorization header if its valid for JWT request.
|
||||
if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
} else if aType == authTypeSTS {
|
||||
h.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
a.handler.ServeHTTP(w, r)
|
||||
return
|
||||
} else if aType == authTypeSTS {
|
||||
a.handler.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
|
||||
writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL, guessIsBrowserReq(r))
|
||||
})
|
||||
}
|
||||
|
||||
func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool, map[string]interface{}, APIErrorCode) {
|
||||
|
||||
@@ -421,7 +421,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
}
|
||||
ctx := context.Background()
|
||||
for i, testCase := range testCases {
|
||||
if _, s3Error := checkAdminRequestAuthType(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
|
||||
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
|
||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,7 +67,7 @@ func waitForLowHTTPReq(maxIO int, maxWait time.Duration) {
|
||||
// Bucket notification and http trace are not costly, it is okay to ignore them
|
||||
// while counting the number of concurrent connections
|
||||
maxIOFn := func() int {
|
||||
return maxIO + globalHTTPListen.NumSubscribers() + globalHTTPTrace.NumSubscribers()
|
||||
return maxIO + int(globalHTTPListen.NumSubscribers()) + int(globalHTTPTrace.NumSubscribers())
|
||||
}
|
||||
|
||||
if httpServer := newHTTPServerFn(); httpServer != nil {
|
||||
|
||||
@@ -153,22 +153,22 @@ wait:
|
||||
continue
|
||||
}
|
||||
|
||||
zoneIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
|
||||
if zoneIdx < 0 {
|
||||
poolIdx := globalEndpoints.GetLocalZoneIdx(disk.Endpoint())
|
||||
if poolIdx < 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Calculate the set index where the current endpoint belongs
|
||||
z.serverPools[zoneIdx].erasureDisksMu.RLock()
|
||||
z.serverPools[poolIdx].erasureDisksMu.RLock()
|
||||
// Protect reading reference format.
|
||||
setIndex, _, err := findDiskIndex(z.serverPools[zoneIdx].format, format)
|
||||
z.serverPools[zoneIdx].erasureDisksMu.RUnlock()
|
||||
setIndex, _, err := findDiskIndex(z.serverPools[poolIdx].format, format)
|
||||
z.serverPools[poolIdx].erasureDisksMu.RUnlock()
|
||||
if err != nil {
|
||||
printEndpointError(endpoint, err, false)
|
||||
continue
|
||||
}
|
||||
|
||||
erasureSetInZoneDisksToHeal[zoneIdx][setIndex] = append(erasureSetInZoneDisksToHeal[zoneIdx][setIndex], disk)
|
||||
erasureSetInZoneDisksToHeal[poolIdx][setIndex] = append(erasureSetInZoneDisksToHeal[poolIdx][setIndex], disk)
|
||||
}
|
||||
|
||||
buckets, _ := z.ListBuckets(ctx)
|
||||
@@ -181,11 +181,11 @@ wait:
|
||||
for i, setMap := range erasureSetInZoneDisksToHeal {
|
||||
for setIndex, disks := range setMap {
|
||||
for _, disk := range disks {
|
||||
logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
|
||||
logger.Info("Healing disk '%s' on %s pool", disk, humanize.Ordinal(i+1))
|
||||
|
||||
// So someone changed the drives underneath, healing tracker missing.
|
||||
if !disk.Healing() {
|
||||
logger.Info("Healing tracker missing on '%s', disk was swapped again on %s zone", disk, humanize.Ordinal(i+1))
|
||||
logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool", disk, humanize.Ordinal(i+1))
|
||||
diskID, err := disk.GetDiskID()
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -209,7 +209,7 @@ wait:
|
||||
continue
|
||||
}
|
||||
|
||||
logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
|
||||
logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
|
||||
|
||||
if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
|
||||
healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
|
||||
|
||||
@@ -55,7 +55,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
for i := 0; i < b.N; i++ {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -114,7 +114,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
|
||||
md5hex := getMD5Hash([]byte(textPartData))
|
||||
var partInfo PartInfo
|
||||
partInfo, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, j,
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -200,7 +200,7 @@ func runGetObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -301,7 +301,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
for pb.Next() {
|
||||
// insert the object.
|
||||
objInfo, err := obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
@@ -340,7 +340,7 @@ func runGetObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
|
||||
// insert the object.
|
||||
var objInfo ObjectInfo
|
||||
objInfo, err = obj.PutObject(context.Background(), bucket, "object"+strconv.Itoa(i),
|
||||
mustGetPutObjReader(b, bytes.NewBuffer(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{})
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
+14
-4
@@ -91,7 +91,8 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
|
||||
// ReadAt() implementation which verifies the bitrot hash available as part of the stream.
|
||||
type streamingBitrotReader struct {
|
||||
disk StorageAPI
|
||||
rc io.ReadCloser
|
||||
data []byte
|
||||
rc io.Reader
|
||||
volume string
|
||||
filePath string
|
||||
tillOffset int64
|
||||
@@ -105,7 +106,10 @@ func (b *streamingBitrotReader) Close() error {
|
||||
if b.rc == nil {
|
||||
return nil
|
||||
}
|
||||
return b.rc.Close()
|
||||
if closer, ok := b.rc.(io.Closer); ok {
|
||||
return closer.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
@@ -119,11 +123,16 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
// For the first ReadAt() call we need to open the stream for reading.
|
||||
b.currOffset = offset
|
||||
streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
|
||||
b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
|
||||
if len(b.data) == 0 {
|
||||
b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
|
||||
} else {
|
||||
b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
|
||||
}
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
if offset != b.currOffset {
|
||||
// Can never happen unless there are programmer bugs
|
||||
return 0, errUnexpected
|
||||
@@ -150,10 +159,11 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
|
||||
}
|
||||
|
||||
// Returns streaming bitrot reader implementation.
|
||||
func newStreamingBitrotReader(disk StorageAPI, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
|
||||
func newStreamingBitrotReader(disk StorageAPI, data []byte, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader {
|
||||
h := algo.New()
|
||||
return &streamingBitrotReader{
|
||||
disk,
|
||||
data,
|
||||
nil,
|
||||
volume,
|
||||
filePath,
|
||||
|
||||
+2
-2
@@ -103,9 +103,9 @@ func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, alg
|
||||
return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
|
||||
}
|
||||
|
||||
func newBitrotReader(disk StorageAPI, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
|
||||
func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt {
|
||||
if algo == HighwayHash256S {
|
||||
return newStreamingBitrotReader(disk, bucket, filePath, tillOffset, algo, shardSize)
|
||||
return newStreamingBitrotReader(disk, data, bucket, filePath, tillOffset, algo, shardSize)
|
||||
}
|
||||
return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum)
|
||||
}
|
||||
|
||||
+1
-1
@@ -62,7 +62,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
|
||||
}
|
||||
writer.(io.Closer).Close()
|
||||
|
||||
reader := newBitrotReader(disk, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
|
||||
reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10)
|
||||
b := make([]byte, 10)
|
||||
if _, err = reader.ReadAt(b, 0); err != nil {
|
||||
log.Fatal(err)
|
||||
|
||||
+18
-16
@@ -33,7 +33,6 @@ import (
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/config/dns"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
@@ -42,7 +41,6 @@ import (
|
||||
objectlock "github.com/minio/minio/pkg/bucket/object/lock"
|
||||
"github.com/minio/minio/pkg/bucket/policy"
|
||||
"github.com/minio/minio/pkg/bucket/replication"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/handlers"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
@@ -407,9 +405,9 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
getObjectInfoFn = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
var (
|
||||
hasLockEnabled, hasLifecycleConfig bool
|
||||
goi ObjectInfo
|
||||
gerr error
|
||||
hasLockEnabled, hasLifecycleConfig, replicateSync bool
|
||||
goi ObjectInfo
|
||||
gerr error
|
||||
)
|
||||
replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjects.Objects)
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled {
|
||||
@@ -457,11 +455,19 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
object.PurgeTransitioned = goi.TransitionStatus
|
||||
}
|
||||
if replicateDeletes {
|
||||
delMarker, replicate := checkReplicateDelete(ctx, bucket, ObjectToDelete{
|
||||
delMarker, replicate, repsync := checkReplicateDelete(ctx, bucket, ObjectToDelete{
|
||||
ObjectName: object.ObjectName,
|
||||
VersionID: object.VersionID,
|
||||
}, goi, gerr)
|
||||
replicateSync = repsync
|
||||
if replicate {
|
||||
if apiErrCode := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object.ObjectName); apiErrCode != ErrNone {
|
||||
if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
continue
|
||||
}
|
||||
if object.VersionID != "" {
|
||||
object.VersionPurgeStatus = Pending
|
||||
if delMarker {
|
||||
@@ -551,10 +557,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
for _, dobj := range deletedObjects {
|
||||
if replicateDeletes {
|
||||
if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
|
||||
globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
|
||||
dv := DeletedObjectVersionInfo{
|
||||
DeletedObject: dobj,
|
||||
Bucket: bucket,
|
||||
})
|
||||
}
|
||||
scheduleReplicationDelete(ctx, dv, objectAPI, replicateSync)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -748,7 +755,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
if !objectAPI.IsEncryptionSupported() && crypto.IsRequested(r.Header) {
|
||||
if _, ok := crypto.IsRequested(r.Header); !objectAPI.IsEncryptionSupported() && ok {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
}
|
||||
@@ -895,7 +902,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
|
||||
// This request header needs to be set prior to setting ObjectOptions
|
||||
if !crypto.SSEC.IsRequested(r.Header) {
|
||||
r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
|
||||
r.Header.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -907,7 +914,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
if objectAPI.IsEncryptionSupported() {
|
||||
if crypto.IsRequested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
|
||||
if _, ok := crypto.IsRequested(formValues); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
|
||||
if crypto.SSECopy.IsRequested(r.Header) {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
@@ -1336,11 +1343,6 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
|
||||
return
|
||||
}
|
||||
// Turn off replication if disk crawl is unavailable.
|
||||
if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOff {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketReplicationDisabledError), r.URL)
|
||||
return
|
||||
}
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL, guessIsBrowserReq(r))
|
||||
return
|
||||
|
||||
@@ -35,7 +35,7 @@ func TestRemoveBucketHandler(t *testing.T) {
|
||||
|
||||
func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
|
||||
credentials auth.Credentials, t *testing.T) {
|
||||
_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewBuffer([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
|
||||
_, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewReader([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{})
|
||||
// if object upload fails stop the test.
|
||||
if err != nil {
|
||||
t.Fatalf("Error uploading object: <ERROR> %v", err)
|
||||
@@ -669,7 +669,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
|
||||
for i := 0; i < 10; i++ {
|
||||
objectName := "test-object-" + strconv.Itoa(i)
|
||||
// uploading the object.
|
||||
_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewBuffer(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
|
||||
_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
|
||||
// if object upload fails stop the test.
|
||||
if err != nil {
|
||||
t.Fatalf("Put Object %d: Error uploading object: <ERROR> %v", i, err)
|
||||
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
sse "github.com/minio/minio/pkg/bucket/encryption"
|
||||
@@ -341,13 +340,14 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
|
||||
return err
|
||||
}
|
||||
oi := gr.ObjInfo
|
||||
|
||||
if oi.TransitionStatus == lifecycle.TransitionComplete {
|
||||
gr.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
putOpts := putTransitionOpts(oi)
|
||||
if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, "", "", putOpts); err != nil {
|
||||
if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, putOpts); err != nil {
|
||||
gr.Close()
|
||||
return err
|
||||
}
|
||||
gr.Close()
|
||||
@@ -421,7 +421,7 @@ func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs
|
||||
}
|
||||
}
|
||||
|
||||
reader, _, _, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
|
||||
reader, err := tgt.GetObject(ctx, arn.Bucket, object, gopts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -547,7 +547,7 @@ func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer)
|
||||
if r.OutputLocation.S3.Prefix == "" {
|
||||
return fmt.Errorf("Prefix is a required parameter in OutputLocation")
|
||||
}
|
||||
if r.OutputLocation.S3.Encryption.EncryptionType != crypto.SSEAlgorithmAES256 {
|
||||
if r.OutputLocation.S3.Encryption.EncryptionType != xhttp.AmzEncryptionAES {
|
||||
return NotImplemented{}
|
||||
}
|
||||
}
|
||||
@@ -573,7 +573,7 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
|
||||
}
|
||||
meta[xhttp.AmzObjectTagging] = rreq.OutputLocation.S3.Tagging.String()
|
||||
if rreq.OutputLocation.S3.Encryption.EncryptionType != "" {
|
||||
meta[crypto.SSEHeader] = crypto.SSEAlgorithmAES256
|
||||
meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||
}
|
||||
return ObjectOptions{
|
||||
Versioned: globalBucketVersioningSys.Enabled(bucket),
|
||||
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
bucketsse "github.com/minio/minio/pkg/bucket/encryption"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
@@ -168,7 +169,10 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
|
||||
}
|
||||
meta.ReplicationConfigXML = configData
|
||||
case bucketTargetsFile:
|
||||
meta.BucketTargetsConfigJSON = configData
|
||||
meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(meta.Name, configData, crypto.Context{bucket: meta.Name, bucketTargetsFile: bucketTargetsFile})
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error encrypting bucket target metadata %w", err)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
}
|
||||
@@ -473,6 +477,15 @@ func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, ob
|
||||
}
|
||||
}
|
||||
|
||||
// Reset the state of the BucketMetadataSys.
|
||||
func (sys *BucketMetadataSys) Reset() {
|
||||
sys.Lock()
|
||||
for k := range sys.metadataMap {
|
||||
delete(sys.metadataMap, k)
|
||||
}
|
||||
sys.Unlock()
|
||||
}
|
||||
|
||||
// NewBucketMetadataSys - creates new policy system.
|
||||
func NewBucketMetadataSys() *BucketMetadataSys {
|
||||
return &BucketMetadataSys{
|
||||
|
||||
+112
-31
@@ -19,6 +19,7 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
@@ -28,6 +29,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
bucketsse "github.com/minio/minio/pkg/bucket/encryption"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
@@ -37,6 +39,7 @@ import (
|
||||
"github.com/minio/minio/pkg/bucket/versioning"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -61,31 +64,33 @@ var (
|
||||
// bucketMetadataFormat refers to the format.
|
||||
// bucketMetadataVersion can be used to track a rolling upgrade of a field.
|
||||
type BucketMetadata struct {
|
||||
Name string
|
||||
Created time.Time
|
||||
LockEnabled bool // legacy not used anymore.
|
||||
PolicyConfigJSON []byte
|
||||
NotificationConfigXML []byte
|
||||
LifecycleConfigXML []byte
|
||||
ObjectLockConfigXML []byte
|
||||
VersioningConfigXML []byte
|
||||
EncryptionConfigXML []byte
|
||||
TaggingConfigXML []byte
|
||||
QuotaConfigJSON []byte
|
||||
ReplicationConfigXML []byte
|
||||
BucketTargetsConfigJSON []byte
|
||||
Name string
|
||||
Created time.Time
|
||||
LockEnabled bool // legacy not used anymore.
|
||||
PolicyConfigJSON []byte
|
||||
NotificationConfigXML []byte
|
||||
LifecycleConfigXML []byte
|
||||
ObjectLockConfigXML []byte
|
||||
VersioningConfigXML []byte
|
||||
EncryptionConfigXML []byte
|
||||
TaggingConfigXML []byte
|
||||
QuotaConfigJSON []byte
|
||||
ReplicationConfigXML []byte
|
||||
BucketTargetsConfigJSON []byte
|
||||
BucketTargetsConfigMetaJSON []byte
|
||||
|
||||
// Unexported fields. Must be updated atomically.
|
||||
policyConfig *policy.Policy
|
||||
notificationConfig *event.Config
|
||||
lifecycleConfig *lifecycle.Lifecycle
|
||||
objectLockConfig *objectlock.Config
|
||||
versioningConfig *versioning.Versioning
|
||||
sseConfig *bucketsse.BucketSSEConfig
|
||||
taggingConfig *tags.Tags
|
||||
quotaConfig *madmin.BucketQuota
|
||||
replicationConfig *replication.Config
|
||||
bucketTargetConfig *madmin.BucketTargets
|
||||
policyConfig *policy.Policy
|
||||
notificationConfig *event.Config
|
||||
lifecycleConfig *lifecycle.Lifecycle
|
||||
objectLockConfig *objectlock.Config
|
||||
versioningConfig *versioning.Versioning
|
||||
sseConfig *bucketsse.BucketSSEConfig
|
||||
taggingConfig *tags.Tags
|
||||
quotaConfig *madmin.BucketQuota
|
||||
replicationConfig *replication.Config
|
||||
bucketTargetConfig *madmin.BucketTargets
|
||||
bucketTargetConfigMeta map[string]string
|
||||
}
|
||||
|
||||
// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now.
|
||||
@@ -100,7 +105,8 @@ func newBucketMetadata(name string) BucketMetadata {
|
||||
versioningConfig: &versioning.Versioning{
|
||||
XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/",
|
||||
},
|
||||
bucketTargetConfig: &madmin.BucketTargets{},
|
||||
bucketTargetConfig: &madmin.BucketTargets{},
|
||||
bucketTargetConfigMeta: make(map[string]string),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,16 +146,16 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
|
||||
func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
|
||||
b := newBucketMetadata(bucket)
|
||||
err := b.Load(ctx, objectAPI, b.Name)
|
||||
if err == nil {
|
||||
return b, b.convertLegacyConfigs(ctx, objectAPI)
|
||||
}
|
||||
|
||||
if !errors.Is(err, errConfigNotFound) {
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
return b, err
|
||||
}
|
||||
|
||||
// Old bucket without bucket metadata. Hence we migrate existing settings.
|
||||
return b, b.convertLegacyConfigs(ctx, objectAPI)
|
||||
if err := b.convertLegacyConfigs(ctx, objectAPI); err != nil {
|
||||
return b, err
|
||||
}
|
||||
// migrate unencrypted remote targets
|
||||
return b, b.migrateTargetConfig(ctx, objectAPI)
|
||||
}
|
||||
|
||||
// parseAllConfigs will parse all configs and populate the private fields.
|
||||
@@ -234,7 +240,8 @@ func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLa
|
||||
}
|
||||
|
||||
if len(b.BucketTargetsConfigJSON) != 0 {
|
||||
if err = json.Unmarshal(b.BucketTargetsConfigJSON, b.bucketTargetConfig); err != nil {
|
||||
b.bucketTargetConfig, err = parseBucketTargetConfig(b.Name, b.BucketTargetsConfigJSON, b.BucketTargetsConfigMetaJSON)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
} else {
|
||||
@@ -354,6 +361,7 @@ func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
configFile := path.Join(bucketConfigPrefix, b.Name, bucketMetadataFile)
|
||||
return saveConfig(ctx, api, configFile, data)
|
||||
}
|
||||
@@ -373,3 +381,76 @@ func deleteBucketMetadata(ctx context.Context, obj objectDeleter, bucket string)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured.
|
||||
func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error {
|
||||
var err error
|
||||
// early return if no targets or already encrypted
|
||||
if len(b.BucketTargetsConfigJSON) == 0 || GlobalKMS == nil || len(b.BucketTargetsConfigMetaJSON) != 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
encBytes, metaBytes, err := encryptBucketMetadata(b.Name, b.BucketTargetsConfigJSON, crypto.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
b.BucketTargetsConfigJSON = encBytes
|
||||
b.BucketTargetsConfigMetaJSON = metaBytes
|
||||
return b.Save(ctx, objectAPI)
|
||||
}
|
||||
|
||||
// encrypt bucket metadata if kms is configured.
|
||||
func encryptBucketMetadata(bucket string, input []byte, kmsContext crypto.Context) (output, metabytes []byte, err error) {
|
||||
var sealedKey crypto.SealedKey
|
||||
if GlobalKMS == nil {
|
||||
output = input
|
||||
return
|
||||
}
|
||||
var (
|
||||
key [32]byte
|
||||
encKey []byte
|
||||
)
|
||||
metadata := make(map[string]string)
|
||||
key, encKey, err = GlobalKMS.GenerateKey(GlobalKMS.DefaultKeyID(), kmsContext)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
outbuf := bytes.NewBuffer(nil)
|
||||
objectKey := crypto.GenerateKey(key, rand.Reader)
|
||||
sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "")
|
||||
crypto.S3.CreateMetadata(metadata, GlobalKMS.DefaultKeyID(), encKey, sealedKey)
|
||||
_, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
|
||||
if err != nil {
|
||||
return output, metabytes, err
|
||||
}
|
||||
metabytes, err = json.Marshal(metadata)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return outbuf.Bytes(), metabytes, nil
|
||||
}
|
||||
|
||||
// decrypt bucket metadata if kms is configured.
|
||||
func decryptBucketMetadata(input []byte, bucket string, meta map[string]string, kmsContext crypto.Context) ([]byte, error) {
|
||||
if GlobalKMS == nil {
|
||||
return nil, errKMSNotConfigured
|
||||
}
|
||||
keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(meta)
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, kmsContext)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var objectKey crypto.ObjectKey
|
||||
if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, ""); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
outbuf := bytes.NewBuffer(nil)
|
||||
_, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20})
|
||||
|
||||
return outbuf.Bytes(), err
|
||||
}
|
||||
|
||||
@@ -102,6 +102,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigJSON")
|
||||
return
|
||||
}
|
||||
case "BucketTargetsConfigMetaJSON":
|
||||
z.BucketTargetsConfigMetaJSON, err = dc.ReadBytes(z.BucketTargetsConfigMetaJSON)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
@@ -115,9 +121,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 13
|
||||
// map header, size 14
|
||||
// write "Name"
|
||||
err = en.Append(0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
err = en.Append(0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -246,15 +252,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigJSON")
|
||||
return
|
||||
}
|
||||
// write "BucketTargetsConfigMetaJSON"
|
||||
err = en.Append(0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteBytes(z.BucketTargetsConfigMetaJSON)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 13
|
||||
// map header, size 14
|
||||
// string "Name"
|
||||
o = append(o, 0x8d, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = append(o, 0x8e, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
// string "Created"
|
||||
o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
|
||||
@@ -292,6 +308,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
// string "BucketTargetsConfigJSON"
|
||||
o = append(o, 0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e)
|
||||
o = msgp.AppendBytes(o, z.BucketTargetsConfigJSON)
|
||||
// string "BucketTargetsConfigMetaJSON"
|
||||
o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e)
|
||||
o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -391,6 +410,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigJSON")
|
||||
return
|
||||
}
|
||||
case "BucketTargetsConfigMetaJSON":
|
||||
z.BucketTargetsConfigMetaJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigMetaJSON)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
@@ -405,6 +430,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *BucketMetadata) Msgsize() (s int) {
|
||||
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON)
|
||||
s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON)
|
||||
return
|
||||
}
|
||||
|
||||
+226
-51
@@ -88,35 +88,37 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
|
||||
return false, BucketRemoteTargetNotFound{Bucket: bucket}
|
||||
}
|
||||
|
||||
func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) bool {
|
||||
func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) (replicate bool, sync bool) {
|
||||
if permErr != ErrNone {
|
||||
return false
|
||||
return
|
||||
}
|
||||
return mustReplicater(ctx, bucket, object, meta, replStatus)
|
||||
}
|
||||
|
||||
// mustReplicate returns true if object meets replication criteria.
|
||||
func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
|
||||
// mustReplicate returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
|
||||
// a synchronous manner.
|
||||
func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) (replicate bool, sync bool) {
|
||||
if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
|
||||
return false
|
||||
return
|
||||
}
|
||||
return mustReplicater(ctx, bucket, object, meta, replStatus)
|
||||
}
|
||||
|
||||
// mustReplicater returns true if object meets replication criteria.
|
||||
func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string) bool {
|
||||
// mustReplicater returns 2 booleans - true if object meets replication criteria and true if replication is to be done in
|
||||
// a synchronous manner.
|
||||
func mustReplicater(ctx context.Context, bucket, object string, meta map[string]string, replStatus string) (replicate bool, sync bool) {
|
||||
if globalIsGateway {
|
||||
return false
|
||||
return replicate, sync
|
||||
}
|
||||
if rs, ok := meta[xhttp.AmzBucketReplicationStatus]; ok {
|
||||
replStatus = rs
|
||||
}
|
||||
if replication.StatusType(replStatus) == replication.Replica {
|
||||
return false
|
||||
return replicate, sync
|
||||
}
|
||||
cfg, err := getReplicationConfig(ctx, bucket)
|
||||
if err != nil {
|
||||
return false
|
||||
return replicate, sync
|
||||
}
|
||||
opts := replication.ObjectOpts{
|
||||
Name: object,
|
||||
@@ -126,7 +128,20 @@ func mustReplicater(ctx context.Context, bucket, object string, meta map[string]
|
||||
if ok {
|
||||
opts.UserTags = tagStr
|
||||
}
|
||||
return cfg.Replicate(opts)
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
|
||||
if tgt == nil || tgt.isOffline() {
|
||||
return cfg.Replicate(opts), false
|
||||
}
|
||||
return cfg.Replicate(opts), tgt.replicateSync
|
||||
}
|
||||
|
||||
// Standard headers that needs to be extracted from User metadata.
|
||||
var standardHeaders = []string{
|
||||
"content-type",
|
||||
"content-encoding",
|
||||
xhttp.AmzStorageClass,
|
||||
xhttp.AmzObjectTagging,
|
||||
xhttp.AmzBucketReplicationStatus,
|
||||
}
|
||||
|
||||
// returns true if any of the objects being deleted qualifies for replication.
|
||||
@@ -143,33 +158,48 @@ func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToD
|
||||
return false
|
||||
}
|
||||
|
||||
// isStandardHeader returns true if header is a supported header and not a custom header
|
||||
func isStandardHeader(headerKey string) bool {
|
||||
key := strings.ToLower(headerKey)
|
||||
for _, header := range standardHeaders {
|
||||
if strings.ToLower(header) == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// returns whether object version is a deletemarker and if object qualifies for replication
|
||||
func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, gerr error) (dm, replicate bool) {
|
||||
func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, gerr error) (dm, replicate, sync bool) {
|
||||
rcfg, err := getReplicationConfig(ctx, bucket)
|
||||
if err != nil || rcfg == nil {
|
||||
return false, false
|
||||
return false, false, sync
|
||||
}
|
||||
// when incoming delete is removal of a delete marker( a.k.a versioned delete),
|
||||
// GetObjectInfo returns extra information even though it returns errFileNotFound
|
||||
if gerr != nil {
|
||||
validReplStatus := false
|
||||
switch oi.ReplicationStatus {
|
||||
case replication.Pending, replication.Complete, replication.Failed:
|
||||
case replication.Pending, replication.Completed, replication.Failed:
|
||||
validReplStatus = true
|
||||
}
|
||||
if oi.DeleteMarker && validReplStatus {
|
||||
return oi.DeleteMarker, true
|
||||
return oi.DeleteMarker, true, sync
|
||||
}
|
||||
return oi.DeleteMarker, false
|
||||
return oi.DeleteMarker, false, sync
|
||||
}
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
|
||||
if tgt == nil || tgt.isOffline() {
|
||||
return oi.DeleteMarker, false, false
|
||||
}
|
||||
opts := replication.ObjectOpts{
|
||||
Name: dobj.ObjectName,
|
||||
SSEC: crypto.SSEC.IsEncrypted(oi.UserDefined),
|
||||
UserTags: oi.UserTags,
|
||||
DeleteMarker: true,
|
||||
DeleteMarker: oi.DeleteMarker,
|
||||
VersionID: dobj.VersionID,
|
||||
}
|
||||
return oi.DeleteMarker, rcfg.Replicate(opts)
|
||||
return oi.DeleteMarker, rcfg.Replicate(opts), tgt.replicateSync
|
||||
}
|
||||
|
||||
// replicate deletes to the designated replication target if replication configuration
|
||||
@@ -216,7 +246,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
|
||||
}
|
||||
} else {
|
||||
if dobj.VersionID == "" {
|
||||
replicationStatus = string(replication.Complete)
|
||||
replicationStatus = string(replication.Completed)
|
||||
} else {
|
||||
versionPurgeStatus = Complete
|
||||
}
|
||||
@@ -287,7 +317,7 @@ func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]
|
||||
if oi.UserTags != "" {
|
||||
meta[xhttp.AmzObjectTagging] = oi.UserTags
|
||||
}
|
||||
meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339)
|
||||
meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
|
||||
meta[xhttp.MinIOSourceETag] = oi.ETag
|
||||
meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
|
||||
return meta
|
||||
@@ -296,10 +326,10 @@ func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]
|
||||
func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
|
||||
meta := make(map[string]string)
|
||||
for k, v := range objInfo.UserDefined {
|
||||
if k == xhttp.AmzBucketReplicationStatus {
|
||||
if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
|
||||
if isStandardHeader(k) {
|
||||
continue
|
||||
}
|
||||
meta[k] = v
|
||||
@@ -330,7 +360,7 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
|
||||
putOpts.Mode = rmode
|
||||
}
|
||||
if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
|
||||
rdate, err := time.Parse(time.RFC3339, retainDateStr)
|
||||
rdate, err := time.Parse(time.RFC3339Nano, retainDateStr)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
@@ -413,6 +443,7 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
|
||||
VersionID: objInfo.VersionID,
|
||||
})
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
}
|
||||
objInfo = gr.ObjInfo
|
||||
@@ -440,38 +471,45 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
|
||||
return
|
||||
}
|
||||
}
|
||||
replicationStatus := replication.Completed
|
||||
if rtype != replicateAll {
|
||||
gr.Close()
|
||||
|
||||
target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
|
||||
return
|
||||
}
|
||||
putOpts := putReplicationOpts(ctx, dest, objInfo)
|
||||
replicationStatus := replication.Complete
|
||||
|
||||
// Setup bandwidth throttling
|
||||
peers, _ := globalEndpoints.peers()
|
||||
totalNodesCount := len(peers)
|
||||
if totalNodesCount == 0 {
|
||||
totalNodesCount = 1 // For standalone erasure coding
|
||||
}
|
||||
b := target.BandwidthLimit / int64(totalNodesCount)
|
||||
var headerSize int
|
||||
for k, v := range putOpts.Header() {
|
||||
headerSize += len(k) + len(v)
|
||||
}
|
||||
r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
|
||||
if rtype == replicateAll {
|
||||
_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
|
||||
} else {
|
||||
// replicate metadata for object tagging/copy with metadata replacement
|
||||
dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
|
||||
_, err = tgt.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
|
||||
}
|
||||
c := &miniogo.Core{Client: tgt.Client}
|
||||
_, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
|
||||
if err != nil {
|
||||
replicationStatus = replication.Failed
|
||||
}
|
||||
} else {
|
||||
target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
|
||||
gr.Close()
|
||||
return
|
||||
}
|
||||
|
||||
r.Close()
|
||||
if err != nil {
|
||||
replicationStatus = replication.Failed
|
||||
putOpts := putReplicationOpts(ctx, dest, objInfo)
|
||||
// Setup bandwidth throttling
|
||||
peers, _ := globalEndpoints.peers()
|
||||
totalNodesCount := len(peers)
|
||||
if totalNodesCount == 0 {
|
||||
totalNodesCount = 1 // For standalone erasure coding
|
||||
}
|
||||
b := target.BandwidthLimit / int64(totalNodesCount)
|
||||
var headerSize int
|
||||
for k, v := range putOpts.Header() {
|
||||
headerSize += len(k) + len(v)
|
||||
}
|
||||
|
||||
// r takes over closing gr.
|
||||
r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
|
||||
_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, putOpts)
|
||||
if err != nil {
|
||||
replicationStatus = replication.Failed
|
||||
}
|
||||
r.Close()
|
||||
}
|
||||
objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
|
||||
if objInfo.UserTags != "" {
|
||||
@@ -616,3 +654,140 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
|
||||
globalReplicationState.addWorker(ctx, objectAPI)
|
||||
}
|
||||
}
|
||||
|
||||
// get Reader from replication target if active-active replication is in place and
|
||||
// this node returns a 404
|
||||
func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, proxy bool) {
|
||||
tgt, oi, proxy, err := proxyHeadToRepTarget(ctx, bucket, object, opts)
|
||||
if !proxy || err != nil {
|
||||
return nil, false
|
||||
}
|
||||
fn, off, length, err := NewGetObjectReader(rs, oi, opts)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
gopts := miniogo.GetObjectOptions{
|
||||
VersionID: opts.VersionID,
|
||||
ServerSideEncryption: opts.ServerSideEncryption,
|
||||
Internal: miniogo.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: true,
|
||||
},
|
||||
}
|
||||
// get correct offsets for encrypted object
|
||||
if off >= 0 && length >= 0 {
|
||||
if err := gopts.SetRange(off, off+length-1); err != nil {
|
||||
return nil, false
|
||||
}
|
||||
}
|
||||
c := miniogo.Core{Client: tgt.Client}
|
||||
|
||||
obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
closeReader := func() { obj.Close() }
|
||||
|
||||
reader, err := fn(obj, h, opts.CheckPrecondFn, closeReader)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
return reader, true
|
||||
}
|
||||
|
||||
// isProxyable returns true if replication config found for this bucket
|
||||
func isProxyable(ctx context.Context, bucket string) bool {
|
||||
cfg, err := getReplicationConfig(ctx, bucket)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
dest := cfg.GetDestination()
|
||||
return dest.Bucket == bucket
|
||||
}
|
||||
func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (tgt *TargetClient, oi ObjectInfo, proxy bool, err error) {
|
||||
// this option is set when active-active replication is in place between site A -> B,
|
||||
// and site B does not have the object yet.
|
||||
if opts.ProxyRequest { // true only when site B sets MinIOSourceProxyRequest header
|
||||
return nil, oi, false, nil
|
||||
}
|
||||
cfg, err := getReplicationConfig(ctx, bucket)
|
||||
if err != nil {
|
||||
return nil, oi, false, err
|
||||
}
|
||||
dest := cfg.GetDestination()
|
||||
if dest.Bucket != bucket { // not active-active
|
||||
return nil, oi, false, err
|
||||
}
|
||||
ssec := false
|
||||
if opts.ServerSideEncryption != nil {
|
||||
ssec = opts.ServerSideEncryption.Type() == encrypt.SSEC
|
||||
}
|
||||
ropts := replication.ObjectOpts{
|
||||
Name: object,
|
||||
SSEC: ssec,
|
||||
}
|
||||
if !cfg.Replicate(ropts) { // no matching rule for object prefix
|
||||
return nil, oi, false, nil
|
||||
}
|
||||
tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
|
||||
if tgt == nil || tgt.isOffline() {
|
||||
return nil, oi, false, fmt.Errorf("missing target")
|
||||
}
|
||||
|
||||
gopts := miniogo.GetObjectOptions{
|
||||
VersionID: opts.VersionID,
|
||||
ServerSideEncryption: opts.ServerSideEncryption,
|
||||
Internal: miniogo.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: true,
|
||||
},
|
||||
}
|
||||
|
||||
objInfo, err := tgt.StatObject(ctx, dest.Bucket, object, gopts)
|
||||
if err != nil {
|
||||
return nil, oi, false, err
|
||||
}
|
||||
tags, _ := tags.MapToObjectTags(objInfo.UserTags)
|
||||
oi = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: object,
|
||||
ModTime: objInfo.LastModified,
|
||||
Size: objInfo.Size,
|
||||
ETag: objInfo.ETag,
|
||||
VersionID: objInfo.VersionID,
|
||||
IsLatest: objInfo.IsLatest,
|
||||
DeleteMarker: objInfo.IsDeleteMarker,
|
||||
ContentType: objInfo.ContentType,
|
||||
Expires: objInfo.Expires,
|
||||
StorageClass: objInfo.StorageClass,
|
||||
ReplicationStatus: replication.StatusType(objInfo.ReplicationStatus),
|
||||
UserDefined: cloneMSS(objInfo.UserMetadata),
|
||||
UserTags: tags.String(),
|
||||
}
|
||||
if ce, ok := oi.UserDefined[xhttp.ContentEncoding]; ok {
|
||||
oi.ContentEncoding = ce
|
||||
delete(oi.UserDefined, xhttp.ContentEncoding)
|
||||
}
|
||||
return tgt, oi, true, nil
|
||||
}
|
||||
|
||||
// get object info from replication target if active-active replication is in place and
|
||||
// this node returns a 404
|
||||
func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, opts ObjectOptions) (oi ObjectInfo, proxy bool, err error) {
|
||||
_, oi, proxy, err = proxyHeadToRepTarget(ctx, bucket, object, opts)
|
||||
return oi, proxy, err
|
||||
}
|
||||
|
||||
func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, sync bool) {
|
||||
if sync {
|
||||
replicateObject(ctx, objInfo, o)
|
||||
} else {
|
||||
globalReplicationState.queueReplicaTask(objInfo)
|
||||
}
|
||||
}
|
||||
|
||||
func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectVersionInfo, o ObjectLayer, sync bool) {
|
||||
if sync {
|
||||
replicateDelete(ctx, dv, o)
|
||||
} else {
|
||||
globalReplicationState.queueReplicaDeleteTask(dv)
|
||||
}
|
||||
}
|
||||
|
||||
+89
-12
@@ -19,23 +19,30 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
"github.com/minio/minio/pkg/bucket/versioning"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
sha256 "github.com/minio/sha256-simd"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultHealthCheckDuration = 60 * time.Second
|
||||
)
|
||||
|
||||
// BucketTargetSys represents bucket targets subsystem
|
||||
type BucketTargetSys struct {
|
||||
sync.RWMutex
|
||||
arnRemotesMap map[string]*miniogo.Core
|
||||
arnRemotesMap map[string]*TargetClient
|
||||
targetsMap map[string][]madmin.BucketTarget
|
||||
}
|
||||
|
||||
@@ -66,7 +73,6 @@ func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType str
|
||||
|
||||
// ListBucketTargets - gets list of bucket targets for this bucket.
|
||||
func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
|
||||
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
|
||||
@@ -130,7 +136,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
|
||||
tgts := sys.targetsMap[bucket]
|
||||
newtgts := make([]madmin.BucketTarget, len(tgts))
|
||||
labels := make(map[string]struct{})
|
||||
labels := make(map[string]struct{}, len(tgts))
|
||||
found := false
|
||||
for idx, t := range tgts {
|
||||
labels[t.Label] = struct{}{}
|
||||
@@ -196,9 +202,12 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
|
||||
// delete ARN type from list of matching targets
|
||||
sys.Lock()
|
||||
defer sys.Unlock()
|
||||
targets := make([]madmin.BucketTarget, 0)
|
||||
found := false
|
||||
tgts := sys.targetsMap[bucket]
|
||||
tgts, ok := sys.targetsMap[bucket]
|
||||
if !ok {
|
||||
return BucketRemoteTargetNotFound{Bucket: bucket}
|
||||
}
|
||||
targets := make([]madmin.BucketTarget, 0, len(tgts))
|
||||
for _, tgt := range tgts {
|
||||
if tgt.Arn != arnStr {
|
||||
targets = append(targets, tgt)
|
||||
@@ -215,7 +224,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
|
||||
}
|
||||
|
||||
// GetRemoteTargetClient returns minio-go client for replication target instance
|
||||
func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
|
||||
func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *TargetClient {
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
return sys.arnRemotesMap[arn]
|
||||
@@ -262,7 +271,7 @@ func (sys *BucketTargetSys) GetRemoteLabelWithArn(ctx context.Context, bucket, a
|
||||
// NewBucketTargetSys - creates new replication system.
|
||||
func NewBucketTargetSys() *BucketTargetSys {
|
||||
return &BucketTargetSys{
|
||||
arnRemotesMap: make(map[string]*miniogo.Core),
|
||||
arnRemotesMap: make(map[string]*TargetClient),
|
||||
targetsMap: make(map[string][]madmin.BucketTarget),
|
||||
}
|
||||
}
|
||||
@@ -343,20 +352,33 @@ var getRemoteTargetInstanceTransport http.RoundTripper
|
||||
var getRemoteTargetInstanceTransportOnce sync.Once
|
||||
|
||||
// Returns a minio-go Client configured to access remote host described in replication target config.
|
||||
func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*miniogo.Core, error) {
|
||||
func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) {
|
||||
config := tcfg.Credentials
|
||||
creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
|
||||
|
||||
getRemoteTargetInstanceTransportOnce.Do(func() {
|
||||
getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour)
|
||||
getRemoteTargetInstanceTransport = newGatewayHTTPTransport(10 * time.Minute)
|
||||
})
|
||||
|
||||
core, err := miniogo.NewCore(tcfg.URL().Host, &miniogo.Options{
|
||||
api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
|
||||
Creds: creds,
|
||||
Secure: tcfg.Secure,
|
||||
Transport: getRemoteTargetInstanceTransport,
|
||||
})
|
||||
return core, err
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hcDuration := defaultHealthCheckDuration
|
||||
if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
|
||||
hcDuration = tcfg.HealthCheckDuration
|
||||
}
|
||||
tc := &TargetClient{
|
||||
Client: api,
|
||||
healthCheckDuration: hcDuration,
|
||||
bucket: tcfg.TargetBucket,
|
||||
replicateSync: tcfg.ReplicationSync,
|
||||
}
|
||||
go tc.healthCheck()
|
||||
return tc, nil
|
||||
}
|
||||
|
||||
// getRemoteARN gets existing ARN for an endpoint or generates a new one.
|
||||
@@ -391,3 +413,58 @@ func generateARN(t *madmin.BucketTarget) string {
|
||||
}
|
||||
return arn.String()
|
||||
}
|
||||
|
||||
// Returns parsed target config. If KMS is configured, remote target is decrypted
|
||||
func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.BucketTargets, error) {
|
||||
var (
|
||||
data []byte
|
||||
err error
|
||||
t madmin.BucketTargets
|
||||
meta map[string]string
|
||||
)
|
||||
if len(cdata) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
data = cdata
|
||||
if len(cmetadata) != 0 {
|
||||
if err := json.Unmarshal(cmetadata, &meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if crypto.S3.IsEncrypted(meta) {
|
||||
if data, err = decryptBucketMetadata(cdata, bucket, meta, crypto.Context{bucket: bucket, bucketTargetsFile: bucketTargetsFile}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err = json.Unmarshal(data, &t); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
// TargetClient is the struct for remote target client.
|
||||
type TargetClient struct {
|
||||
*miniogo.Client
|
||||
up int32
|
||||
healthCheckDuration time.Duration
|
||||
bucket string // remote bucket target
|
||||
replicateSync bool
|
||||
}
|
||||
|
||||
func (tc *TargetClient) isOffline() bool {
|
||||
return atomic.LoadInt32(&tc.up) == 0
|
||||
}
|
||||
|
||||
func (tc *TargetClient) healthCheck() {
|
||||
for {
|
||||
_, err := tc.BucketExists(GlobalContext, tc.bucket)
|
||||
if err != nil {
|
||||
atomic.StoreInt32(&tc.up, 0)
|
||||
time.Sleep(tc.healthCheckDuration * time.Second)
|
||||
continue
|
||||
}
|
||||
atomic.StoreInt32(&tc.up, 1)
|
||||
time.Sleep(tc.healthCheckDuration * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -51,6 +51,11 @@ func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, erro
|
||||
return globalBucketMetadataSys.GetVersioningConfig(bucket)
|
||||
}
|
||||
|
||||
// Reset BucketVersioningSys to initial state.
|
||||
func (sys *BucketVersioningSys) Reset() {
|
||||
// There is currently no internal state.
|
||||
}
|
||||
|
||||
// NewBucketVersioningSys - creates new versioning system.
|
||||
func NewBucketVersioningSys() *BucketVersioningSys {
|
||||
return &BucketVersioningSys{}
|
||||
|
||||
+22
-1
@@ -51,7 +51,7 @@ func init() {
|
||||
logger.RegisterError(config.FmtError)
|
||||
|
||||
rand.Seed(time.Now().UTC().UnixNano())
|
||||
globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
|
||||
globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second, logger.LogOnceIf)
|
||||
|
||||
initGlobalContext()
|
||||
|
||||
@@ -286,6 +286,16 @@ func handleCommonEnvVars() {
|
||||
globalConfigEncrypted = true
|
||||
}
|
||||
|
||||
if env.IsSet(config.EnvRootUser) || env.IsSet(config.EnvRootPassword) {
|
||||
cred, err := auth.CreateCredentials(env.Get(config.EnvRootUser, ""), env.Get(config.EnvRootPassword, ""))
|
||||
if err != nil {
|
||||
logger.Fatal(config.ErrInvalidCredentials(err),
|
||||
"Unable to validate credentials inherited from the shell environment")
|
||||
}
|
||||
globalActiveCred = cred
|
||||
globalConfigEncrypted = true
|
||||
}
|
||||
|
||||
if env.IsSet(config.EnvAccessKeyOld) && env.IsSet(config.EnvSecretKeyOld) {
|
||||
oldCred, err := auth.CreateCredentials(env.Get(config.EnvAccessKeyOld, ""), env.Get(config.EnvSecretKeyOld, ""))
|
||||
if err != nil {
|
||||
@@ -296,6 +306,17 @@ func handleCommonEnvVars() {
|
||||
os.Unsetenv(config.EnvAccessKeyOld)
|
||||
os.Unsetenv(config.EnvSecretKeyOld)
|
||||
}
|
||||
|
||||
if env.IsSet(config.EnvRootUserOld) && env.IsSet(config.EnvRootPasswordOld) {
|
||||
oldCred, err := auth.CreateCredentials(env.Get(config.EnvRootUserOld, ""), env.Get(config.EnvRootPasswordOld, ""))
|
||||
if err != nil {
|
||||
logger.Fatal(config.ErrInvalidCredentials(err),
|
||||
"Unable to validate the old credentials inherited from the shell environment")
|
||||
}
|
||||
globalOldCred = oldCred
|
||||
os.Unsetenv(config.EnvRootUserOld)
|
||||
os.Unsetenv(config.EnvRootPasswordOld)
|
||||
}
|
||||
}
|
||||
|
||||
func logStartupMessage(msg string) {
|
||||
|
||||
@@ -207,7 +207,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
|
||||
}
|
||||
|
||||
if encrypted && globalActiveCred.IsValid() && globalOldCred.IsValid() {
|
||||
logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
|
||||
logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
|
||||
}
|
||||
|
||||
return saveKeyEtcd(ctx, client, backendEncryptedFile, backendEncryptedMigrationComplete)
|
||||
@@ -294,7 +294,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
|
||||
}
|
||||
|
||||
if encrypted && globalActiveCred.IsValid() && activeCredOld.IsValid() {
|
||||
logger.Info("Rotation complete, please make sure to unset MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD envs")
|
||||
logger.Info("Rotation complete, please make sure to unset MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD envs")
|
||||
}
|
||||
|
||||
return saveConfig(GlobalContext, objAPI, backendEncryptedFile, backendEncryptedMigrationComplete)
|
||||
|
||||
@@ -113,8 +113,6 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
|
||||
// acceptable quorum expected for list operations
|
||||
func (sCfg Config) GetListQuorum() int {
|
||||
switch sCfg.ListQuorum {
|
||||
case "optimal":
|
||||
return 3
|
||||
case "reduced":
|
||||
return 2
|
||||
case "disk":
|
||||
@@ -123,7 +121,7 @@ func (sCfg Config) GetListQuorum() int {
|
||||
case "strict":
|
||||
return -1
|
||||
}
|
||||
// Defaults to 3 drives per set.
|
||||
// Defaults to 3 drives per set, defaults to "optimal" value
|
||||
return 3
|
||||
}
|
||||
|
||||
|
||||
@@ -26,19 +26,22 @@ import (
|
||||
|
||||
// Config represents the compression settings.
|
||||
type Config struct {
|
||||
Enabled bool `json:"enabled"`
|
||||
Extensions []string `json:"extensions"`
|
||||
MimeTypes []string `json:"mime-types"`
|
||||
Enabled bool `json:"enabled"`
|
||||
AllowEncrypted bool `json:"allow_encryption"`
|
||||
Extensions []string `json:"extensions"`
|
||||
MimeTypes []string `json:"mime-types"`
|
||||
}
|
||||
|
||||
// Compression environment variables
|
||||
const (
|
||||
Extensions = "extensions"
|
||||
MimeTypes = "mime_types"
|
||||
Extensions = "extensions"
|
||||
AllowEncrypted = "allow_encryption"
|
||||
MimeTypes = "mime_types"
|
||||
|
||||
EnvCompressState = "MINIO_COMPRESS_ENABLE"
|
||||
EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
|
||||
EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES"
|
||||
EnvCompressState = "MINIO_COMPRESS_ENABLE"
|
||||
EnvCompressAllowEncryption = "MINIO_COMPRESS_ALLOW_ENCRYPTION"
|
||||
EnvCompressExtensions = "MINIO_COMPRESS_EXTENSIONS"
|
||||
EnvCompressMimeTypes = "MINIO_COMPRESS_MIME_TYPES"
|
||||
|
||||
// Include-list for compression.
|
||||
DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
|
||||
@@ -52,6 +55,10 @@ var (
|
||||
Key: config.Enable,
|
||||
Value: config.EnableOff,
|
||||
},
|
||||
config.KV{
|
||||
Key: AllowEncrypted,
|
||||
Value: config.EnableOff,
|
||||
},
|
||||
config.KV{
|
||||
Key: Extensions,
|
||||
Value: DefaultExtensions,
|
||||
@@ -101,6 +108,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
allowEnc := env.Get(EnvCompressAllowEncryption, kvs.Get(AllowEncrypted))
|
||||
cfg.AllowEncrypted, err = config.ParseBool(allowEnc)
|
||||
if err != nil {
|
||||
return cfg, err
|
||||
}
|
||||
|
||||
compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions))
|
||||
compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes))
|
||||
compressMimeTypesLegacy := env.Get(EnvCompressMimeTypesLegacy, kvs.Get(MimeTypes))
|
||||
|
||||
+15
-11
@@ -23,17 +23,21 @@ const (
|
||||
|
||||
// Top level common ENVs
|
||||
const (
|
||||
EnvAccessKey = "MINIO_ACCESS_KEY"
|
||||
EnvSecretKey = "MINIO_SECRET_KEY"
|
||||
EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
|
||||
EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
|
||||
EnvBrowser = "MINIO_BROWSER"
|
||||
EnvDomain = "MINIO_DOMAIN"
|
||||
EnvRegionName = "MINIO_REGION_NAME"
|
||||
EnvPublicIPs = "MINIO_PUBLIC_IPS"
|
||||
EnvFSOSync = "MINIO_FS_OSYNC"
|
||||
EnvArgs = "MINIO_ARGS"
|
||||
EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
|
||||
EnvAccessKey = "MINIO_ACCESS_KEY"
|
||||
EnvSecretKey = "MINIO_SECRET_KEY"
|
||||
EnvRootUser = "MINIO_ROOT_USER"
|
||||
EnvRootPassword = "MINIO_ROOT_PASSWORD"
|
||||
EnvAccessKeyOld = "MINIO_ACCESS_KEY_OLD"
|
||||
EnvSecretKeyOld = "MINIO_SECRET_KEY_OLD"
|
||||
EnvRootUserOld = "MINIO_ROOT_USER_OLD"
|
||||
EnvRootPasswordOld = "MINIO_ROOT_PASSWORD_OLD"
|
||||
EnvBrowser = "MINIO_BROWSER"
|
||||
EnvDomain = "MINIO_DOMAIN"
|
||||
EnvRegionName = "MINIO_REGION_NAME"
|
||||
EnvPublicIPs = "MINIO_PUBLIC_IPS"
|
||||
EnvFSOSync = "MINIO_FS_OSYNC"
|
||||
EnvArgs = "MINIO_ARGS"
|
||||
EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
|
||||
|
||||
EnvUpdate = "MINIO_UPDATE"
|
||||
|
||||
|
||||
@@ -116,19 +116,19 @@ var (
|
||||
ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
|
||||
"Invalid rotating credentials",
|
||||
"Please set correct rotating credentials in the environment for decryption",
|
||||
`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ACCESS_KEY_OLD and MINIO_SECRET_KEY_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
|
||||
`Detected encrypted config backend, correct old access and secret keys should be specified via environment variables MINIO_ROOT_USER_OLD and MINIO_ROOT_PASSWORD_OLD to be able to re-encrypt the MinIO config, user IAM and policies with new credentials`,
|
||||
)
|
||||
|
||||
ErrInvalidCredentialsBackendEncrypted = newErrFn(
|
||||
"Invalid credentials",
|
||||
"Please set correct credentials in the environment for decryption",
|
||||
`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
|
||||
`Detected encrypted config backend, correct access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
|
||||
)
|
||||
|
||||
ErrMissingCredentialsBackendEncrypted = newErrFn(
|
||||
"Credentials missing",
|
||||
"Please set your credentials in the environment",
|
||||
`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY to be able to decrypt the MinIO config, user IAM and policies`,
|
||||
`Detected encrypted config backend, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD to be able to decrypt the MinIO config, user IAM and policies`,
|
||||
)
|
||||
|
||||
ErrInvalidCredentials = newErrFn(
|
||||
@@ -140,13 +140,13 @@ var (
|
||||
ErrEnvCredentialsMissingGateway = newErrFn(
|
||||
"Credentials missing",
|
||||
"Please set your credentials in the environment",
|
||||
`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
|
||||
`In Gateway mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
|
||||
)
|
||||
|
||||
ErrEnvCredentialsMissingDistributed = newErrFn(
|
||||
"Credentials missing",
|
||||
"Please set your credentials in the environment",
|
||||
`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ACCESS_KEY and MINIO_SECRET_KEY respectively`,
|
||||
`In distributed server mode, access and secret keys should be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`,
|
||||
)
|
||||
|
||||
ErrInvalidErasureEndpoints = newErrFn(
|
||||
|
||||
@@ -70,7 +70,7 @@ func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointServerPools EndpointServerP
|
||||
// HasLogListeners returns true if console log listeners are registered
|
||||
// for this node or peers
|
||||
func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
|
||||
return sys != nil && sys.pubsub.HasSubscribers()
|
||||
return sys != nil && sys.pubsub.NumSubscribers() > 0
|
||||
}
|
||||
|
||||
// Subscribe starts console logging for this node.
|
||||
|
||||
+10
-179
@@ -19,231 +19,62 @@ import (
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
)
|
||||
|
||||
// SSEHeader is the general AWS SSE HTTP header key.
|
||||
const SSEHeader = "X-Amz-Server-Side-Encryption"
|
||||
|
||||
const (
|
||||
// SSEKmsID is the HTTP header key referencing the SSE-KMS
|
||||
// key ID.
|
||||
SSEKmsID = SSEHeader + "-Aws-Kms-Key-Id"
|
||||
|
||||
// SSEKmsContext is the HTTP header key referencing the
|
||||
// SSE-KMS encryption context.
|
||||
SSEKmsContext = SSEHeader + "-Context"
|
||||
)
|
||||
|
||||
const (
|
||||
// SSECAlgorithm is the HTTP header key referencing
|
||||
// the SSE-C algorithm.
|
||||
SSECAlgorithm = SSEHeader + "-Customer-Algorithm"
|
||||
|
||||
// SSECKey is the HTTP header key referencing the
|
||||
// SSE-C client-provided key..
|
||||
SSECKey = SSEHeader + "-Customer-Key"
|
||||
|
||||
// SSECKeyMD5 is the HTTP header key referencing
|
||||
// the MD5 sum of the client-provided key.
|
||||
SSECKeyMD5 = SSEHeader + "-Customer-Key-Md5"
|
||||
)
|
||||
|
||||
const (
|
||||
// SSECopyAlgorithm is the HTTP header key referencing
|
||||
// the SSE-C algorithm for SSE-C copy requests.
|
||||
SSECopyAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm"
|
||||
|
||||
// SSECopyKey is the HTTP header key referencing the SSE-C
|
||||
// client-provided key for SSE-C copy requests.
|
||||
SSECopyKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key"
|
||||
|
||||
// SSECopyKeyMD5 is the HTTP header key referencing the
|
||||
// MD5 sum of the client key for SSE-C copy requests.
|
||||
SSECopyKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5"
|
||||
)
|
||||
|
||||
const (
|
||||
// SSEAlgorithmAES256 is the only supported value for the SSE-S3 or SSE-C algorithm header.
|
||||
// For SSE-S3 see: https://docs.aws.amazon.com/AmazonS3/latest/dev/SSEUsingRESTAPI.html
|
||||
// For SSE-C see: https://docs.aws.amazon.com/AmazonS3/latest/dev/ServerSideEncryptionCustomerKeys.html
|
||||
SSEAlgorithmAES256 = "AES256"
|
||||
|
||||
// SSEAlgorithmKMS is the value of 'X-Amz-Server-Side-Encryption' for SSE-KMS.
|
||||
// See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingKMSEncryption.html
|
||||
SSEAlgorithmKMS = "aws:kms"
|
||||
)
|
||||
|
||||
// RemoveSensitiveHeaders removes confidential encryption
|
||||
// information - e.g. the SSE-C key - from the HTTP headers.
|
||||
// It has the same semantics as RemoveSensitiveEntires.
|
||||
func RemoveSensitiveHeaders(h http.Header) {
|
||||
h.Del(SSECKey)
|
||||
h.Del(SSECopyKey)
|
||||
h.Del(xhttp.AmzServerSideEncryptionCustomerKey)
|
||||
h.Del(xhttp.AmzServerSideEncryptionCopyCustomerKey)
|
||||
h.Del(xhttp.AmzMetaUnencryptedContentLength)
|
||||
h.Del(xhttp.AmzMetaUnencryptedContentMD5)
|
||||
}
|
||||
|
||||
// IsRequested returns true if the HTTP headers indicates
|
||||
// that any form server-side encryption (SSE-C, SSE-S3 or SSE-KMS)
|
||||
// is requested.
|
||||
func IsRequested(h http.Header) bool {
|
||||
return S3.IsRequested(h) || SSEC.IsRequested(h) || SSECopy.IsRequested(h) || S3KMS.IsRequested(h)
|
||||
}
|
||||
|
||||
// S3 represents AWS SSE-S3. It provides functionality to handle
|
||||
// SSE-S3 requests.
|
||||
var S3 = s3{}
|
||||
|
||||
type s3 struct{}
|
||||
|
||||
// IsRequested returns true if the HTTP headers indicates that
|
||||
// the S3 client requests SSE-S3.
|
||||
func (s3) IsRequested(h http.Header) bool {
|
||||
_, ok := h[SSEHeader]
|
||||
return ok && strings.ToLower(h.Get(SSEHeader)) != SSEAlgorithmKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-S3 related HTTP headers and checks
|
||||
// whether they contain valid values.
|
||||
func (s3) ParseHTTP(h http.Header) (err error) {
|
||||
if h.Get(SSEHeader) != SSEAlgorithmAES256 {
|
||||
err = ErrInvalidEncryptionMethod
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// S3KMS represents AWS SSE-KMS. It provides functionality to
|
||||
// handle SSE-KMS requests.
|
||||
var S3KMS = s3KMS{}
|
||||
|
||||
type s3KMS struct{}
|
||||
|
||||
// IsRequested returns true if the HTTP headers indicates that
|
||||
// the S3 client requests SSE-KMS.
|
||||
func (s3KMS) IsRequested(h http.Header) bool {
|
||||
if _, ok := h[SSEKmsID]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSEKmsContext]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSEHeader]; ok {
|
||||
return strings.ToUpper(h.Get(SSEHeader)) != SSEAlgorithmAES256 // Return only true if the SSE header is specified and does not contain the SSE-S3 value
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
|
||||
// and context, if present, on success.
|
||||
func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
|
||||
algorithm := h.Get(SSEHeader)
|
||||
if algorithm != SSEAlgorithmKMS {
|
||||
return "", nil, ErrInvalidEncryptionMethod
|
||||
}
|
||||
|
||||
contextStr, ok := h[SSEKmsContext]
|
||||
if ok {
|
||||
var context map[string]interface{}
|
||||
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
return h.Get(SSEKmsID), context, nil
|
||||
}
|
||||
return h.Get(SSEKmsID), nil, nil
|
||||
}
|
||||
|
||||
var (
|
||||
// SSEC represents AWS SSE-C. It provides functionality to handle
|
||||
// SSE-C requests.
|
||||
SSEC = ssec{}
|
||||
|
||||
// SSECopy represents AWS SSE-C for copy requests. It provides
|
||||
// functionality to handle SSE-C copy requests.
|
||||
SSECopy = ssecCopy{}
|
||||
)
|
||||
|
||||
type ssec struct{}
|
||||
type ssecCopy struct{}
|
||||
|
||||
// IsRequested returns true if the HTTP headers contains
|
||||
// at least one SSE-C header. SSE-C copy headers are ignored.
|
||||
func (ssec) IsRequested(h http.Header) bool {
|
||||
if _, ok := h[SSECAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECKey]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECKeyMD5]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsRequested returns true if the HTTP headers contains
|
||||
// at least one SSE-C copy header. Regular SSE-C headers
|
||||
// are ignored.
|
||||
func (ssecCopy) IsRequested(h http.Header) bool {
|
||||
if _, ok := h[SSECopyAlgorithm]; ok {
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECopyKey]; ok {
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKey]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[SSECopyKeyMD5]; ok {
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
|
||||
// on success. SSE-C copy headers are ignored.
|
||||
func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
|
||||
if h.Get(SSECAlgorithm) != SSEAlgorithmAES256 {
|
||||
return key, ErrInvalidCustomerAlgorithm
|
||||
}
|
||||
if h.Get(SSECKey) == "" {
|
||||
return key, ErrMissingCustomerKey
|
||||
}
|
||||
if h.Get(SSECKeyMD5) == "" {
|
||||
return key, ErrMissingCustomerKeyMD5
|
||||
}
|
||||
|
||||
clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECKey))
|
||||
if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
|
||||
return key, ErrInvalidCustomerKey
|
||||
}
|
||||
keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECKeyMD5))
|
||||
if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
|
||||
return key, ErrCustomerKeyMD5Mismatch
|
||||
}
|
||||
copy(key[:], clientKey)
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key
|
||||
// on success. Regular SSE-C headers are ignored.
|
||||
func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) {
|
||||
if h.Get(SSECopyAlgorithm) != SSEAlgorithmAES256 {
|
||||
if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != xhttp.AmzEncryptionAES {
|
||||
return key, ErrInvalidCustomerAlgorithm
|
||||
}
|
||||
if h.Get(SSECopyKey) == "" {
|
||||
if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey) == "" {
|
||||
return key, ErrMissingCustomerKey
|
||||
}
|
||||
if h.Get(SSECopyKeyMD5) == "" {
|
||||
if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5) == "" {
|
||||
return key, ErrMissingCustomerKeyMD5
|
||||
}
|
||||
|
||||
clientKey, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKey))
|
||||
clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey))
|
||||
if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
|
||||
return key, ErrInvalidCustomerKey
|
||||
}
|
||||
keyMD5, err := base64.StdEncoding.DecodeString(h.Get(SSECopyKeyMD5))
|
||||
keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5))
|
||||
if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
|
||||
return key, ErrCustomerKeyMD5Mismatch
|
||||
}
|
||||
|
||||
+39
-36
@@ -18,26 +18,29 @@ import (
|
||||
"net/http"
|
||||
"sort"
|
||||
"testing"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
)
|
||||
|
||||
func TestIsRequested(t *testing.T) {
|
||||
for i, test := range kmsIsRequestedTests {
|
||||
if got := IsRequested(test.Header) && S3KMS.IsRequested(test.Header); got != test.Expected {
|
||||
_, got := IsRequested(test.Header)
|
||||
got = got && S3KMS.IsRequested(test.Header)
|
||||
if got != test.Expected {
|
||||
t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
for i, test := range s3IsRequestedTests {
|
||||
if got := IsRequested(test.Header) && S3.IsRequested(test.Header); got != test.Expected {
|
||||
_, got := IsRequested(test.Header)
|
||||
got = got && S3.IsRequested(test.Header)
|
||||
if got != test.Expected {
|
||||
t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
for i, test := range ssecIsRequestedTests {
|
||||
if got := IsRequested(test.Header) && SSEC.IsRequested(test.Header); got != test.Expected {
|
||||
t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
for i, test := range ssecCopyIsRequestedTests {
|
||||
if got := IsRequested(test.Header) && SSECopy.IsRequested(test.Header); got != test.Expected {
|
||||
_, got := IsRequested(test.Header)
|
||||
got = got && SSEC.IsRequested(test.Header)
|
||||
if got != test.Expected {
|
||||
t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got)
|
||||
}
|
||||
}
|
||||
@@ -131,11 +134,11 @@ var s3IsRequestedTests = []struct {
|
||||
Header http.Header
|
||||
Expected bool
|
||||
}{
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true}, // 0
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true}, // 2
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false}, // 3
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{SSEAlgorithmKMS}}, Expected: false}, // 4
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true}, // 0
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true}, // 2
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false}, // 3
|
||||
{Header: http.Header{"X-Amz-Server-Side-Encryption": []string{xhttp.AmzEncryptionKMS}}, Expected: false}, // 4
|
||||
}
|
||||
|
||||
func TestS3IsRequested(t *testing.T) {
|
||||
@@ -403,7 +406,7 @@ func TestSSECopyParse(t *testing.T) {
|
||||
if err == nil && key == zeroKey {
|
||||
t.Errorf("Test %d: parsed client key is zero key", i)
|
||||
}
|
||||
if _, ok := test.Header[SSECKey]; ok {
|
||||
if _, ok := test.Header[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
|
||||
t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i)
|
||||
}
|
||||
}
|
||||
@@ -414,47 +417,47 @@ var removeSensitiveHeadersTests = []struct {
|
||||
}{
|
||||
{
|
||||
Header: http.Header{
|
||||
SSECKey: []string{""},
|
||||
SSECopyKey: []string{""},
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: []string{""},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{""},
|
||||
},
|
||||
ExpectedHeader: http.Header{},
|
||||
},
|
||||
{ // Standard SSE-C request headers
|
||||
Header: http.Header{
|
||||
SSECAlgorithm: []string{SSEAlgorithmAES256},
|
||||
SSECKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedHeader: http.Header{
|
||||
SSECAlgorithm: []string{SSEAlgorithmAES256},
|
||||
SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
},
|
||||
{ // Standard SSE-C + SSE-C-copy request headers
|
||||
Header: http.Header{
|
||||
SSECAlgorithm: []string{SSEAlgorithmAES256},
|
||||
SSECKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
SSECopyKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
ExpectedHeader: http.Header{
|
||||
SSECAlgorithm: []string{SSEAlgorithmAES256},
|
||||
SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
},
|
||||
},
|
||||
{ // Standard SSE-C + metadata request headers
|
||||
Header: http.Header{
|
||||
SSECAlgorithm: []string{SSEAlgorithmAES256},
|
||||
SSECKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
"X-Amz-Meta-Test-1": []string{"Test-1"},
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
"X-Amz-Meta-Test-1": []string{"Test-1"},
|
||||
},
|
||||
ExpectedHeader: http.Header{
|
||||
SSECAlgorithm: []string{SSEAlgorithmAES256},
|
||||
SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
"X-Amz-Meta-Test-1": []string{"Test-1"},
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="},
|
||||
"X-Amz-Meta-Test-1": []string{"Test-1"},
|
||||
},
|
||||
},
|
||||
{ // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
||||
|
||||
+55
-196
@@ -15,19 +15,42 @@
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
const (
|
||||
// MetaMultipart indicates that the object has been uploaded
|
||||
// in multiple parts - via the S3 multipart API.
|
||||
MetaMultipart = "X-Minio-Internal-Encrypted-Multipart"
|
||||
|
||||
// MetaIV is the random initialization vector (IV) used for
|
||||
// the MinIO-internal key derivation.
|
||||
MetaIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
|
||||
|
||||
// MetaAlgorithm is the algorithm used to derive internal keys
|
||||
// and encrypt the objects.
|
||||
MetaAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
|
||||
|
||||
// MetaSealedKeySSEC is the sealed object encryption key in case of SSE-C.
|
||||
MetaSealedKeySSEC = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
|
||||
// MetaSealedKeyS3 is the sealed object encryption key in case of SSE-S3
|
||||
MetaSealedKeyS3 = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
|
||||
// MetaSealedKeyKMS is the sealed object encryption key in case of SSE-KMS
|
||||
MetaSealedKeyKMS = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key"
|
||||
|
||||
// MetaKeyID is the KMS master key ID used to generate/encrypt the data
|
||||
// encryption key (DEK).
|
||||
MetaKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
|
||||
// MetaDataEncryptionKey is the sealed data encryption key (DEK) received from
|
||||
// the KMS.
|
||||
MetaDataEncryptionKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
|
||||
)
|
||||
|
||||
// IsMultiPart returns true if the object metadata indicates
|
||||
// that it was uploaded using some form of server-side-encryption
|
||||
// and the S3 multipart API.
|
||||
func IsMultiPart(metadata map[string]string) bool {
|
||||
if _, ok := metadata[SSEMultipart]; ok {
|
||||
if _, ok := metadata[MetaMultipart]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -37,8 +60,8 @@ func IsMultiPart(metadata map[string]string) bool {
|
||||
// information - e.g. the SSE-C key - from the metadata map.
|
||||
// It has the same semantics as RemoveSensitiveHeaders.
|
||||
func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
|
||||
delete(metadata, SSECKey)
|
||||
delete(metadata, SSECopyKey)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
|
||||
delete(metadata, xhttp.AmzMetaUnencryptedContentLength)
|
||||
delete(metadata, xhttp.AmzMetaUnencryptedContentMD5)
|
||||
}
|
||||
@@ -46,31 +69,36 @@ func RemoveSensitiveEntries(metadata map[string]string) { // The functions is te
|
||||
// RemoveSSEHeaders removes all crypto-specific SSE
|
||||
// header entries from the metadata map.
|
||||
func RemoveSSEHeaders(metadata map[string]string) {
|
||||
delete(metadata, SSEHeader)
|
||||
delete(metadata, SSEKmsID)
|
||||
delete(metadata, SSEKmsContext)
|
||||
delete(metadata, SSECKeyMD5)
|
||||
delete(metadata, SSECAlgorithm)
|
||||
delete(metadata, xhttp.AmzServerSideEncryption)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionKmsID)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionKmsContext)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCustomerAlgorithm)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCustomerKeyMD5)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey)
|
||||
delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5)
|
||||
}
|
||||
|
||||
// RemoveInternalEntries removes all crypto-specific internal
|
||||
// metadata entries from the metadata map.
|
||||
func RemoveInternalEntries(metadata map[string]string) {
|
||||
delete(metadata, SSEMultipart)
|
||||
delete(metadata, SSEIV)
|
||||
delete(metadata, SSESealAlgorithm)
|
||||
delete(metadata, SSECSealedKey)
|
||||
delete(metadata, S3SealedKey)
|
||||
delete(metadata, S3KMSKeyID)
|
||||
delete(metadata, S3KMSSealedKey)
|
||||
delete(metadata, MetaMultipart)
|
||||
delete(metadata, MetaAlgorithm)
|
||||
delete(metadata, MetaIV)
|
||||
delete(metadata, MetaSealedKeySSEC)
|
||||
delete(metadata, MetaSealedKeyS3)
|
||||
delete(metadata, MetaSealedKeyKMS)
|
||||
delete(metadata, MetaKeyID)
|
||||
delete(metadata, MetaDataEncryptionKey)
|
||||
}
|
||||
|
||||
// IsSourceEncrypted returns true if the source is encrypted
|
||||
func IsSourceEncrypted(metadata map[string]string) bool {
|
||||
if _, ok := metadata[SSECAlgorithm]; ok {
|
||||
if _, ok := metadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[SSEHeader]; ok {
|
||||
if _, ok := metadata[xhttp.AmzServerSideEncryption]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -82,10 +110,10 @@ func IsSourceEncrypted(metadata map[string]string) bool {
|
||||
// IsEncrypted only checks whether the metadata contains at least
|
||||
// one entry indicating SSE-C or SSE-S3.
|
||||
func IsEncrypted(metadata map[string]string) bool {
|
||||
if _, ok := metadata[SSEIV]; ok {
|
||||
if _, ok := metadata[MetaIV]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[SSESealAlgorithm]; ok {
|
||||
if _, ok := metadata[MetaAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if IsMultiPart(metadata) {
|
||||
@@ -97,28 +125,7 @@ func IsEncrypted(metadata map[string]string) bool {
|
||||
if SSEC.IsEncrypted(metadata) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncrypted returns true if the object metadata indicates
|
||||
// that the object was uploaded using SSE-S3.
|
||||
func (s3) IsEncrypted(metadata map[string]string) bool {
|
||||
if _, ok := metadata[S3SealedKey]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[S3KMSKeyID]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[S3KMSSealedKey]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncrypted returns true if the object metadata indicates
|
||||
// that the object was uploaded using SSE-C.
|
||||
func (ssec) IsEncrypted(metadata map[string]string) bool {
|
||||
if _, ok := metadata[SSECSealedKey]; ok {
|
||||
if S3KMS.IsEncrypted(metadata) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -129,159 +136,11 @@ func (ssec) IsEncrypted(metadata map[string]string) bool {
|
||||
// metadata is nil.
|
||||
func CreateMultipartMetadata(metadata map[string]string) map[string]string {
|
||||
if metadata == nil {
|
||||
return map[string]string{SSEMultipart: ""}
|
||||
return map[string]string{MetaMultipart: ""}
|
||||
}
|
||||
metadata[SSEMultipart] = ""
|
||||
metadata[MetaMultipart] = ""
|
||||
return metadata
|
||||
}
|
||||
|
||||
// CreateMetadata encodes the sealed object key into the metadata and returns
|
||||
// the modified metadata. If the keyID and the kmsKey is not empty it encodes
|
||||
// both into the metadata as well. It allocates a new metadata map if metadata
|
||||
// is nil.
|
||||
func (s3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
|
||||
if sealedKey.Algorithm != SealAlgorithm {
|
||||
logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
|
||||
}
|
||||
|
||||
// There are two possibilites:
|
||||
// - We use a KMS -> There must be non-empty key ID and a KMS data key.
|
||||
// - We use a K/V -> There must be no key ID and no KMS data key.
|
||||
// Otherwise, the caller has passed an invalid argument combination.
|
||||
if keyID == "" && len(kmsKey) != 0 {
|
||||
logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
|
||||
}
|
||||
if keyID != "" && len(kmsKey) == 0 {
|
||||
logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 5)
|
||||
}
|
||||
|
||||
metadata[SSESealAlgorithm] = sealedKey.Algorithm
|
||||
metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
|
||||
metadata[S3SealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
|
||||
if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
|
||||
metadata[S3KMSKeyID] = keyID
|
||||
metadata[S3KMSSealedKey] = base64.StdEncoding.EncodeToString(kmsKey)
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
// ParseMetadata extracts all SSE-S3 related values from the object metadata
|
||||
// and checks whether they are well-formed. It returns the sealed object key
|
||||
// on success. If the metadata contains both, a KMS master key ID and a sealed
|
||||
// KMS data key it returns both. If the metadata does not contain neither a
|
||||
// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
|
||||
// KMS data key. Otherwise, it returns an error.
|
||||
func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
|
||||
// Extract all required values from object metadata
|
||||
b64IV, ok := metadata[SSEIV]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, errMissingInternalIV
|
||||
}
|
||||
algorithm, ok := metadata[SSESealAlgorithm]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
|
||||
}
|
||||
b64SealedKey, ok := metadata[S3SealedKey]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
|
||||
}
|
||||
|
||||
// There are two possibilites:
|
||||
// - We use a KMS -> There must be a key ID and a KMS data key.
|
||||
// - We use a K/V -> There must be no key ID and no KMS data key.
|
||||
// Otherwise, the metadata is corrupted.
|
||||
keyID, idPresent := metadata[S3KMSKeyID]
|
||||
b64KMSSealedKey, kmsKeyPresent := metadata[S3KMSSealedKey]
|
||||
if !idPresent && kmsKeyPresent {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
|
||||
}
|
||||
if idPresent && !kmsKeyPresent {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
|
||||
}
|
||||
|
||||
// Check whether all extracted values are well-formed
|
||||
var iv [32]byte
|
||||
n, err := base64.StdEncoding.Decode(iv[:], []byte(b64IV))
|
||||
if err != nil || n != 32 {
|
||||
return keyID, kmsKey, sealedKey, errInvalidInternalIV
|
||||
}
|
||||
if algorithm != SealAlgorithm {
|
||||
return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
|
||||
}
|
||||
var encryptedKey [64]byte
|
||||
n, err = base64.StdEncoding.Decode(encryptedKey[:], []byte(b64SealedKey))
|
||||
if err != nil || n != 64 {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
|
||||
}
|
||||
if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
|
||||
kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
|
||||
if err != nil {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
sealedKey.Algorithm = algorithm
|
||||
sealedKey.IV = iv
|
||||
sealedKey.Key = encryptedKey
|
||||
return keyID, kmsKey, sealedKey, nil
|
||||
}
|
||||
|
||||
// CreateMetadata encodes the sealed key into the metadata and returns the modified metadata.
|
||||
// It allocates a new metadata map if metadata is nil.
|
||||
func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
|
||||
if sealedKey.Algorithm != SealAlgorithm {
|
||||
logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 3)
|
||||
}
|
||||
metadata[SSESealAlgorithm] = SealAlgorithm
|
||||
metadata[SSEIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
|
||||
metadata[SSECSealedKey] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
|
||||
return metadata
|
||||
}
|
||||
|
||||
// ParseMetadata extracts all SSE-C related values from the object metadata
|
||||
// and checks whether they are well-formed. It returns the sealed object key
|
||||
// on success.
|
||||
func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
|
||||
// Extract all required values from object metadata
|
||||
b64IV, ok := metadata[SSEIV]
|
||||
if !ok {
|
||||
return sealedKey, errMissingInternalIV
|
||||
}
|
||||
algorithm, ok := metadata[SSESealAlgorithm]
|
||||
if !ok {
|
||||
return sealedKey, errMissingInternalSealAlgorithm
|
||||
}
|
||||
b64SealedKey, ok := metadata[SSECSealedKey]
|
||||
if !ok {
|
||||
return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
|
||||
}
|
||||
|
||||
// Check whether all extracted values are well-formed
|
||||
iv, err := base64.StdEncoding.DecodeString(b64IV)
|
||||
if err != nil || len(iv) != 32 {
|
||||
return sealedKey, errInvalidInternalIV
|
||||
}
|
||||
if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
|
||||
return sealedKey, errInvalidInternalSealAlgorithm
|
||||
}
|
||||
encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
|
||||
if err != nil || len(encryptedKey) != 64 {
|
||||
return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
|
||||
}
|
||||
|
||||
sealedKey.Algorithm = algorithm
|
||||
copy(sealedKey.IV[:], iv)
|
||||
copy(sealedKey.Key[:], encryptedKey)
|
||||
return sealedKey, nil
|
||||
}
|
||||
|
||||
// IsETagSealed returns true if the etag seems to be encrypted.
|
||||
func IsETagSealed(etag []byte) bool { return len(etag) > 16 }
|
||||
|
||||
+60
-60
@@ -27,9 +27,9 @@ var isMultipartTests = []struct {
|
||||
Metadata map[string]string
|
||||
Multipart bool
|
||||
}{
|
||||
{Multipart: true, Metadata: map[string]string{SSEMultipart: ""}}, // 0
|
||||
{Multipart: true, Metadata: map[string]string{MetaMultipart: ""}}, // 0
|
||||
{Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1
|
||||
{Multipart: true, Metadata: map[string]string{SSEMultipart: "some-value"}}, // 2
|
||||
{Multipart: true, Metadata: map[string]string{MetaMultipart: "some-value"}}, // 2
|
||||
{Multipart: false, Metadata: map[string]string{"": ""}}, // 3
|
||||
{Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4
|
||||
}
|
||||
@@ -46,13 +46,13 @@ var isEncryptedTests = []struct {
|
||||
Metadata map[string]string
|
||||
Encrypted bool
|
||||
}{
|
||||
{Encrypted: true, Metadata: map[string]string{SSEMultipart: ""}}, // 0
|
||||
{Encrypted: true, Metadata: map[string]string{SSEIV: ""}}, // 1
|
||||
{Encrypted: true, Metadata: map[string]string{SSESealAlgorithm: ""}}, // 2
|
||||
{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}}, // 3
|
||||
{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}}, // 4
|
||||
{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}}, // 5
|
||||
{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}}, // 6
|
||||
{Encrypted: true, Metadata: map[string]string{MetaMultipart: ""}}, // 0
|
||||
{Encrypted: true, Metadata: map[string]string{MetaIV: ""}}, // 1
|
||||
{Encrypted: true, Metadata: map[string]string{MetaAlgorithm: ""}}, // 2
|
||||
{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}}, // 3
|
||||
{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}}, // 4
|
||||
{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}}, // 5
|
||||
{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}}, // 6
|
||||
{Encrypted: false, Metadata: map[string]string{"": ""}}, // 7
|
||||
{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
|
||||
}
|
||||
@@ -69,13 +69,13 @@ var s3IsEncryptedTests = []struct {
|
||||
Metadata map[string]string
|
||||
Encrypted bool
|
||||
}{
|
||||
{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}}, // 0
|
||||
{Encrypted: false, Metadata: map[string]string{SSEIV: ""}}, // 1
|
||||
{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}}, // 2
|
||||
{Encrypted: false, Metadata: map[string]string{SSECSealedKey: ""}}, // 3
|
||||
{Encrypted: true, Metadata: map[string]string{S3SealedKey: ""}}, // 4
|
||||
{Encrypted: true, Metadata: map[string]string{S3KMSKeyID: ""}}, // 5
|
||||
{Encrypted: true, Metadata: map[string]string{S3KMSSealedKey: ""}}, // 6
|
||||
{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}}, // 0
|
||||
{Encrypted: false, Metadata: map[string]string{MetaIV: ""}}, // 1
|
||||
{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}}, // 2
|
||||
{Encrypted: false, Metadata: map[string]string{MetaSealedKeySSEC: ""}}, // 3
|
||||
{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}}, // 4
|
||||
{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}}, // 5
|
||||
{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}}, // 6
|
||||
{Encrypted: false, Metadata: map[string]string{"": ""}}, // 7
|
||||
{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
|
||||
}
|
||||
@@ -92,13 +92,13 @@ var ssecIsEncryptedTests = []struct {
|
||||
Metadata map[string]string
|
||||
Encrypted bool
|
||||
}{
|
||||
{Encrypted: false, Metadata: map[string]string{SSEMultipart: ""}}, // 0
|
||||
{Encrypted: false, Metadata: map[string]string{SSEIV: ""}}, // 1
|
||||
{Encrypted: false, Metadata: map[string]string{SSESealAlgorithm: ""}}, // 2
|
||||
{Encrypted: true, Metadata: map[string]string{SSECSealedKey: ""}}, // 3
|
||||
{Encrypted: false, Metadata: map[string]string{S3SealedKey: ""}}, // 4
|
||||
{Encrypted: false, Metadata: map[string]string{S3KMSKeyID: ""}}, // 5
|
||||
{Encrypted: false, Metadata: map[string]string{S3KMSSealedKey: ""}}, // 6
|
||||
{Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}}, // 0
|
||||
{Encrypted: false, Metadata: map[string]string{MetaIV: ""}}, // 1
|
||||
{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}}, // 2
|
||||
{Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}}, // 3
|
||||
{Encrypted: false, Metadata: map[string]string{MetaSealedKeyS3: ""}}, // 4
|
||||
{Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}}, // 5
|
||||
{Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}}, // 6
|
||||
{Encrypted: false, Metadata: map[string]string{"": ""}}, // 7
|
||||
{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
|
||||
}
|
||||
@@ -121,65 +121,65 @@ var s3ParseMetadataTests = []struct {
|
||||
}{
|
||||
{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0
|
||||
{
|
||||
ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""},
|
||||
ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""},
|
||||
DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
}, // 1
|
||||
{
|
||||
ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"),
|
||||
Metadata: map[string]string{SSEIV: "", SSESealAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
Metadata: map[string]string{MetaIV: "", MetaAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
}, // 2
|
||||
{
|
||||
ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"),
|
||||
Metadata: map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSSealedKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaDataEncryptionKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
}, // 3
|
||||
{
|
||||
ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"),
|
||||
Metadata: map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: ""},
|
||||
Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: ""},
|
||||
DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
}, // 4
|
||||
{
|
||||
ExpectedErr: errInvalidInternalIV,
|
||||
Metadata: map[string]string{SSEIV: "", SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: ""},
|
||||
Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: ""},
|
||||
DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
}, // 5
|
||||
{
|
||||
ExpectedErr: errInvalidInternalSealAlgorithm,
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", S3SealedKey: "", S3KMSKeyID: "", S3KMSSealedKey: "",
|
||||
MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: "",
|
||||
},
|
||||
DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
}, // 6
|
||||
{
|
||||
ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"),
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, S3SealedKey: "",
|
||||
S3KMSKeyID: "", S3KMSSealedKey: "",
|
||||
MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeyS3: "",
|
||||
MetaKeyID: "", MetaDataEncryptionKey: "",
|
||||
},
|
||||
DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{},
|
||||
}, // 7
|
||||
{
|
||||
ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"),
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
|
||||
S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "key-1",
|
||||
S3KMSSealedKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
|
||||
MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
|
||||
MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "key-1",
|
||||
MetaDataEncryptionKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64
|
||||
},
|
||||
DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{},
|
||||
}, // 8
|
||||
{
|
||||
ExpectedErr: nil,
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
|
||||
S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)), S3KMSKeyID: "", S3KMSSealedKey: "",
|
||||
MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
|
||||
MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "", MetaDataEncryptionKey: "",
|
||||
},
|
||||
DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm},
|
||||
}, // 9
|
||||
{
|
||||
ExpectedErr: nil,
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: SealAlgorithm,
|
||||
S3SealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), S3KMSKeyID: "key-1",
|
||||
S3KMSSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
|
||||
MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: SealAlgorithm,
|
||||
MetaSealedKeyS3: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), MetaKeyID: "key-1",
|
||||
MetaDataEncryptionKey: base64.StdEncoding.EncodeToString(make([]byte, 48)),
|
||||
},
|
||||
DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
|
||||
}, // 10
|
||||
@@ -223,43 +223,43 @@ var ssecParseMetadataTests = []struct {
|
||||
|
||||
SealedKey SealedKey
|
||||
}{
|
||||
{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}}, // 0
|
||||
{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{SSEIV: ""}, SealedKey: SealedKey{}}, // 1
|
||||
{ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}}, // 0
|
||||
{ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""}, SealedKey: SealedKey{}}, // 1
|
||||
{
|
||||
ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"),
|
||||
Metadata: map[string]string{SSEIV: "", SSESealAlgorithm: ""}, SealedKey: SealedKey{},
|
||||
Metadata: map[string]string{MetaIV: "", MetaAlgorithm: ""}, SealedKey: SealedKey{},
|
||||
}, // 2
|
||||
{
|
||||
ExpectedErr: errInvalidInternalIV,
|
||||
Metadata: map[string]string{SSEIV: "", SSESealAlgorithm: "", SSECSealedKey: ""}, SealedKey: SealedKey{},
|
||||
Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeySSEC: ""}, SealedKey: SealedKey{},
|
||||
}, // 3
|
||||
{
|
||||
ExpectedErr: errInvalidInternalSealAlgorithm,
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: "", SSECSealedKey: "",
|
||||
MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeySSEC: "",
|
||||
},
|
||||
SealedKey: SealedKey{},
|
||||
}, // 4
|
||||
{
|
||||
ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"),
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm, SSECSealedKey: "",
|
||||
MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeySSEC: "",
|
||||
},
|
||||
SealedKey: SealedKey{},
|
||||
}, // 5
|
||||
{
|
||||
ExpectedErr: nil,
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), SSESealAlgorithm: SealAlgorithm,
|
||||
SSECSealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm,
|
||||
MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
},
|
||||
SealedKey: SealedKey{Algorithm: SealAlgorithm},
|
||||
}, // 6
|
||||
{
|
||||
ExpectedErr: nil,
|
||||
Metadata: map[string]string{
|
||||
SSEIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), SSESealAlgorithm: InsecureSealAlgorithm,
|
||||
SSECSealedKey: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
|
||||
MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: InsecureSealAlgorithm,
|
||||
MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)),
|
||||
},
|
||||
SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}},
|
||||
}, // 7
|
||||
@@ -267,8 +267,8 @@ var ssecParseMetadataTests = []struct {
|
||||
|
||||
func TestCreateMultipartMetadata(t *testing.T) {
|
||||
metadata := CreateMultipartMetadata(nil)
|
||||
if v, ok := metadata[SSEMultipart]; !ok || v != "" {
|
||||
t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", SSEMultipart, v, "")
|
||||
if v, ok := metadata[MetaMultipart]; !ok || v != "" {
|
||||
t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", MetaMultipart, v, "")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,20 +411,20 @@ var removeInternalEntriesTests = []struct {
|
||||
}{
|
||||
{ // 0
|
||||
Metadata: map[string]string{
|
||||
SSEMultipart: "",
|
||||
SSEIV: "",
|
||||
SSESealAlgorithm: "",
|
||||
SSECSealedKey: "",
|
||||
S3SealedKey: "",
|
||||
S3KMSKeyID: "",
|
||||
S3KMSSealedKey: "",
|
||||
MetaMultipart: "",
|
||||
MetaIV: "",
|
||||
MetaAlgorithm: "",
|
||||
MetaSealedKeySSEC: "",
|
||||
MetaSealedKeyS3: "",
|
||||
MetaKeyID: "",
|
||||
MetaDataEncryptionKey: "",
|
||||
},
|
||||
Expected: map[string]string{},
|
||||
},
|
||||
{ // 1
|
||||
Metadata: map[string]string{
|
||||
SSEMultipart: "",
|
||||
SSEIV: "",
|
||||
MetaMultipart: "",
|
||||
MetaIV: "",
|
||||
"X-Amz-Meta-A": "X",
|
||||
"X-Minio-Internal-B": "Y",
|
||||
},
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/base64"
|
||||
"net/http"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
type ssec struct{}
|
||||
|
||||
var (
|
||||
// SSEC represents AWS SSE-C. It provides functionality to handle
|
||||
// SSE-C requests.
|
||||
SSEC = ssec{}
|
||||
|
||||
_ Type = SSEC
|
||||
)
|
||||
|
||||
// String returns the SSE domain as string. For SSE-C the
|
||||
// domain is "SSE-C".
|
||||
func (ssec) String() string { return "SSE-C" }
|
||||
|
||||
// IsRequested returns true if the HTTP headers contains
|
||||
// at least one SSE-C header. SSE-C copy headers are ignored.
|
||||
func (ssec) IsRequested(h http.Header) bool {
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKey]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKeyMD5]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// IsEncrypted returns true if the metadata contains an SSE-C
|
||||
// entry inidicating that the object has been encrypted using
|
||||
// SSE-C.
|
||||
func (ssec) IsEncrypted(metadata map[string]string) bool {
|
||||
if _, ok := metadata[MetaSealedKeySSEC]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-C headers and returns the SSE-C client key
|
||||
// on success. SSE-C copy headers are ignored.
|
||||
func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) {
|
||||
if h.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm) != xhttp.AmzEncryptionAES {
|
||||
return key, ErrInvalidCustomerAlgorithm
|
||||
}
|
||||
if h.Get(xhttp.AmzServerSideEncryptionCustomerKey) == "" {
|
||||
return key, ErrMissingCustomerKey
|
||||
}
|
||||
if h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5) == "" {
|
||||
return key, ErrMissingCustomerKeyMD5
|
||||
}
|
||||
|
||||
clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKey))
|
||||
if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long
|
||||
return key, ErrInvalidCustomerKey
|
||||
}
|
||||
keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
|
||||
if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) {
|
||||
return key, ErrCustomerKeyMD5Mismatch
|
||||
}
|
||||
copy(key[:], clientKey)
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using the SSE-C client key of the HTTP headers
|
||||
// and returns the decrypted object key.
|
||||
func (s3 ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
clientKey, err := s3.ParseHTTP(h)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return unsealObjectKey(clientKey, metadata, bucket, object)
|
||||
}
|
||||
|
||||
// CreateMetadata encodes the sealed key into the metadata
|
||||
// and returns the modified metadata. It allocates a new
|
||||
// metadata map if metadata is nil.
|
||||
func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string {
|
||||
if sealedKey.Algorithm != SealAlgorithm {
|
||||
logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm))
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 3)
|
||||
}
|
||||
metadata[MetaAlgorithm] = SealAlgorithm
|
||||
metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
|
||||
metadata[MetaSealedKeySSEC] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
|
||||
return metadata
|
||||
}
|
||||
|
||||
// ParseMetadata extracts all SSE-C related values from the object metadata
|
||||
// and checks whether they are well-formed. It returns the sealed object key
|
||||
// on success.
|
||||
func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) {
|
||||
// Extract all required values from object metadata
|
||||
b64IV, ok := metadata[MetaIV]
|
||||
if !ok {
|
||||
return sealedKey, errMissingInternalIV
|
||||
}
|
||||
algorithm, ok := metadata[MetaAlgorithm]
|
||||
if !ok {
|
||||
return sealedKey, errMissingInternalSealAlgorithm
|
||||
}
|
||||
b64SealedKey, ok := metadata[MetaSealedKeySSEC]
|
||||
if !ok {
|
||||
return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C")
|
||||
}
|
||||
|
||||
// Check whether all extracted values are well-formed
|
||||
iv, err := base64.StdEncoding.DecodeString(b64IV)
|
||||
if err != nil || len(iv) != 32 {
|
||||
return sealedKey, errInvalidInternalIV
|
||||
}
|
||||
if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm {
|
||||
return sealedKey, errInvalidInternalSealAlgorithm
|
||||
}
|
||||
encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
|
||||
if err != nil || len(encryptedKey) != 64 {
|
||||
return sealedKey, Errorf("The internal sealed key for SSE-C is invalid")
|
||||
}
|
||||
|
||||
sealedKey.Algorithm = algorithm
|
||||
copy(sealedKey.IV[:], iv)
|
||||
copy(sealedKey.Key[:], encryptedKey)
|
||||
return sealedKey, nil
|
||||
}
|
||||
@@ -0,0 +1,201 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
type ssekms struct{}
|
||||
|
||||
var (
|
||||
// S3KMS represents AWS SSE-KMS. It provides functionality to
|
||||
// handle SSE-KMS requests.
|
||||
S3KMS = ssekms{}
|
||||
|
||||
_ Type = S3KMS
|
||||
)
|
||||
|
||||
// String returns the SSE domain as string. For SSE-KMS the
|
||||
// domain is "SSE-KMS".
|
||||
func (ssekms) String() string { return "SSE-KMS" }
|
||||
|
||||
// IsRequested returns true if the HTTP headers contains
|
||||
// at least one SSE-KMS header.
|
||||
func (ssekms) IsRequested(h http.Header) bool {
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionKmsID]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := h[xhttp.AmzServerSideEncryption]; ok {
|
||||
return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES // Return only true if the SSE header is specified and does not contain the SSE-S3 value
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID
|
||||
// and the KMS context on success.
|
||||
func (ssekms) ParseHTTP(h http.Header) (string, Context, error) {
|
||||
algorithm := h.Get(xhttp.AmzServerSideEncryption)
|
||||
if algorithm != xhttp.AmzEncryptionKMS {
|
||||
return "", nil, ErrInvalidEncryptionMethod
|
||||
}
|
||||
|
||||
var ctx Context
|
||||
if context, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok {
|
||||
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
if err := json.Unmarshal([]byte(context[0]), &ctx); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
return h.Get(xhttp.AmzServerSideEncryptionKmsID), ctx, nil
|
||||
}
|
||||
|
||||
// IsEncrypted returns true if the object metadata indicates
|
||||
// that the object was uploaded using SSE-KMS.
|
||||
func (ssekms) IsEncrypted(metadata map[string]string) bool {
|
||||
if _, ok := metadata[MetaSealedKeyKMS]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[MetaKeyID]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[MetaDataEncryptionKey]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using KMS and returns the decrypted object
|
||||
// key.
|
||||
func (s3 ssekms) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
|
||||
return key, err
|
||||
}
|
||||
|
||||
// CreateMetadata encodes the sealed object key into the metadata and returns
|
||||
// the modified metadata. If the keyID and the kmsKey is not empty it encodes
|
||||
// both into the metadata as well. It allocates a new metadata map if metadata
|
||||
// is nil.
|
||||
func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
|
||||
if sealedKey.Algorithm != SealAlgorithm {
|
||||
logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
|
||||
}
|
||||
|
||||
// There are two possibilites:
|
||||
// - We use a KMS -> There must be non-empty key ID and a KMS data key.
|
||||
// - We use a K/V -> There must be no key ID and no KMS data key.
|
||||
// Otherwise, the caller has passed an invalid argument combination.
|
||||
if keyID == "" && len(kmsKey) != 0 {
|
||||
logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
|
||||
}
|
||||
if keyID != "" && len(kmsKey) == 0 {
|
||||
logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 5)
|
||||
}
|
||||
|
||||
metadata[MetaAlgorithm] = sealedKey.Algorithm
|
||||
metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
|
||||
metadata[MetaSealedKeyKMS] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
|
||||
if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
|
||||
metadata[MetaKeyID] = keyID
|
||||
metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
// ParseMetadata extracts all SSE-KMS related values from the object metadata
|
||||
// and checks whether they are well-formed. It returns the sealed object key
|
||||
// on success. If the metadata contains both, a KMS master key ID and a sealed
|
||||
// KMS data key it returns both. If the metadata does not contain neither a
|
||||
// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
|
||||
// KMS data key. Otherwise, it returns an error.
|
||||
func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
|
||||
// Extract all required values from object metadata
|
||||
b64IV, ok := metadata[MetaIV]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, errMissingInternalIV
|
||||
}
|
||||
algorithm, ok := metadata[MetaAlgorithm]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
|
||||
}
|
||||
b64SealedKey, ok := metadata[MetaSealedKeyKMS]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
|
||||
}
|
||||
|
||||
// There are two possibilites:
|
||||
// - We use a KMS -> There must be a key ID and a KMS data key.
|
||||
// - We use a K/V -> There must be no key ID and no KMS data key.
|
||||
// Otherwise, the metadata is corrupted.
|
||||
keyID, idPresent := metadata[MetaKeyID]
|
||||
b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
|
||||
if !idPresent && kmsKeyPresent {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
|
||||
}
|
||||
if idPresent && !kmsKeyPresent {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
|
||||
}
|
||||
|
||||
// Check whether all extracted values are well-formed
|
||||
iv, err := base64.StdEncoding.DecodeString(b64IV)
|
||||
if err != nil || len(iv) != 32 {
|
||||
return keyID, kmsKey, sealedKey, errInvalidInternalIV
|
||||
}
|
||||
if algorithm != SealAlgorithm {
|
||||
return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
|
||||
}
|
||||
encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
|
||||
if err != nil || len(encryptedKey) != 64 {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
|
||||
}
|
||||
if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
|
||||
kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
|
||||
if err != nil {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
sealedKey.Algorithm = algorithm
|
||||
copy(sealedKey.IV[:], iv)
|
||||
copy(sealedKey.Key[:], encryptedKey)
|
||||
return keyID, kmsKey, sealedKey, nil
|
||||
}
|
||||
@@ -0,0 +1,181 @@
|
||||
/*
|
||||
* Minio Cloud Storage, (C) 2019-2020 Minio, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package crypto
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"net/http"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
)
|
||||
|
||||
type sses3 struct{}
|
||||
|
||||
var (
|
||||
// S3 represents AWS SSE-S3. It provides functionality to handle
|
||||
// SSE-S3 requests.
|
||||
S3 = sses3{}
|
||||
|
||||
_ Type = S3
|
||||
)
|
||||
|
||||
// String returns the SSE domain as string. For SSE-S3 the
|
||||
// domain is "SSE-S3".
|
||||
func (sses3) String() string { return "SSE-S3" }
|
||||
|
||||
func (sses3) IsRequested(h http.Header) bool {
|
||||
_, ok := h[xhttp.AmzServerSideEncryption]
|
||||
return ok && strings.ToLower(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionKMS // Return only true if the SSE header is specified and does not contain the SSE-KMS value
|
||||
}
|
||||
|
||||
// ParseHTTP parses the SSE-S3 related HTTP headers and checks
|
||||
// whether they contain valid values.
|
||||
func (sses3) ParseHTTP(h http.Header) error {
|
||||
if h.Get(xhttp.AmzServerSideEncryption) != xhttp.AmzEncryptionAES {
|
||||
return ErrInvalidEncryptionMethod
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// IsEncrypted returns true if the object metadata indicates
|
||||
// that the object was uploaded using SSE-S3.
|
||||
func (sses3) IsEncrypted(metadata map[string]string) bool {
|
||||
if _, ok := metadata[MetaSealedKeyS3]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[MetaKeyID]; ok {
|
||||
return true
|
||||
}
|
||||
if _, ok := metadata[MetaDataEncryptionKey]; ok {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using KMS and returns the decrypted object
|
||||
// key.
|
||||
func (s3 sses3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return key, err
|
||||
}
|
||||
err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object)
|
||||
return key, err
|
||||
}
|
||||
|
||||
// CreateMetadata encodes the sealed object key into the metadata and returns
|
||||
// the modified metadata. If the keyID and the kmsKey is not empty it encodes
|
||||
// both into the metadata as well. It allocates a new metadata map if metadata
|
||||
// is nil.
|
||||
func (sses3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string {
|
||||
if sealedKey.Algorithm != SealAlgorithm {
|
||||
logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm))
|
||||
}
|
||||
|
||||
// There are two possibilites:
|
||||
// - We use a KMS -> There must be non-empty key ID and a KMS data key.
|
||||
// - We use a K/V -> There must be no key ID and no KMS data key.
|
||||
// Otherwise, the caller has passed an invalid argument combination.
|
||||
if keyID == "" && len(kmsKey) != 0 {
|
||||
logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present"))
|
||||
}
|
||||
if keyID != "" && len(kmsKey) == 0 {
|
||||
logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present"))
|
||||
}
|
||||
|
||||
if metadata == nil {
|
||||
metadata = make(map[string]string, 5)
|
||||
}
|
||||
|
||||
metadata[MetaAlgorithm] = sealedKey.Algorithm
|
||||
metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:])
|
||||
metadata[MetaSealedKeyS3] = base64.StdEncoding.EncodeToString(sealedKey.Key[:])
|
||||
if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key.
|
||||
metadata[MetaKeyID] = keyID
|
||||
metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey)
|
||||
}
|
||||
return metadata
|
||||
}
|
||||
|
||||
// ParseMetadata extracts all SSE-S3 related values from the object metadata
|
||||
// and checks whether they are well-formed. It returns the sealed object key
|
||||
// on success. If the metadata contains both, a KMS master key ID and a sealed
|
||||
// KMS data key it returns both. If the metadata does not contain neither a
|
||||
// KMS master key ID nor a sealed KMS data key it returns an empty keyID and
|
||||
// KMS data key. Otherwise, it returns an error.
|
||||
func (sses3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
|
||||
// Extract all required values from object metadata
|
||||
b64IV, ok := metadata[MetaIV]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, errMissingInternalIV
|
||||
}
|
||||
algorithm, ok := metadata[MetaAlgorithm]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
|
||||
}
|
||||
b64SealedKey, ok := metadata[MetaSealedKeyS3]
|
||||
if !ok {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
|
||||
}
|
||||
|
||||
// There are two possibilites:
|
||||
// - We use a KMS -> There must be a key ID and a KMS data key.
|
||||
// - We use a K/V -> There must be no key ID and no KMS data key.
|
||||
// Otherwise, the metadata is corrupted.
|
||||
keyID, idPresent := metadata[MetaKeyID]
|
||||
b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
|
||||
if !idPresent && kmsKeyPresent {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
|
||||
}
|
||||
if idPresent && !kmsKeyPresent {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
|
||||
}
|
||||
|
||||
// Check whether all extracted values are well-formed
|
||||
iv, err := base64.StdEncoding.DecodeString(b64IV)
|
||||
if err != nil || len(iv) != 32 {
|
||||
return keyID, kmsKey, sealedKey, errInvalidInternalIV
|
||||
}
|
||||
if algorithm != SealAlgorithm {
|
||||
return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
|
||||
}
|
||||
encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
|
||||
if err != nil || len(encryptedKey) != 64 {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
|
||||
}
|
||||
if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
|
||||
kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
|
||||
if err != nil {
|
||||
return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
sealedKey.Algorithm = algorithm
|
||||
copy(sealedKey.IV[:], iv)
|
||||
copy(sealedKey.Key[:], encryptedKey)
|
||||
return keyID, kmsKey, sealedKey, nil
|
||||
}
|
||||
+25
-59
@@ -17,44 +17,15 @@ package crypto
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/ioutil"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
const (
|
||||
// SSEMultipart is the metadata key indicating that the object
|
||||
// was uploaded using the S3 multipart API and stored using
|
||||
// some from of server-side-encryption.
|
||||
SSEMultipart = "X-Minio-Internal-Encrypted-Multipart"
|
||||
|
||||
// SSEIV is the metadata key referencing the random initialization
|
||||
// vector (IV) used for SSE-S3 and SSE-C key derivation.
|
||||
SSEIV = "X-Minio-Internal-Server-Side-Encryption-Iv"
|
||||
|
||||
// SSESealAlgorithm is the metadata key referencing the algorithm
|
||||
// used by SSE-C and SSE-S3 to encrypt the object.
|
||||
SSESealAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm"
|
||||
|
||||
// SSECSealedKey is the metadata key referencing the sealed object-key for SSE-C.
|
||||
SSECSealedKey = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key"
|
||||
|
||||
// S3SealedKey is the metadata key referencing the sealed object-key for SSE-S3.
|
||||
S3SealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key"
|
||||
|
||||
// S3KMSKeyID is the metadata key referencing the KMS key-id used to
|
||||
// generate/decrypt the S3-KMS-Sealed-Key. It is only used for SSE-S3 + KMS.
|
||||
S3KMSKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id"
|
||||
|
||||
// S3KMSSealedKey is the metadata key referencing the encrypted key generated
|
||||
// by KMS. It is only used for SSE-S3 + KMS.
|
||||
S3KMSSealedKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
|
||||
)
|
||||
|
||||
const (
|
||||
// SealAlgorithm is the encryption/sealing algorithm used to derive & seal
|
||||
// the key-encryption-key and to en/decrypt the object data.
|
||||
@@ -67,39 +38,34 @@ const (
|
||||
InsecureSealAlgorithm = "DARE-SHA256"
|
||||
)
|
||||
|
||||
// String returns the SSE domain as string. For SSE-S3 the
|
||||
// domain is "SSE-S3".
|
||||
func (s3) String() string { return "SSE-S3" }
|
||||
// Type represents an AWS SSE type:
|
||||
// • SSE-C
|
||||
// • SSE-S3
|
||||
// • SSE-KMS
|
||||
type Type interface {
|
||||
fmt.Stringer
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using KMS and returns the decrypted object
|
||||
// key.
|
||||
func (sse s3) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
keyID, kmsKey, sealedKey, err := sse.ParseMetadata(metadata)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = key.Unseal(unsealKey, sealedKey, sse.String(), bucket, object)
|
||||
return
|
||||
IsRequested(http.Header) bool
|
||||
|
||||
IsEncrypted(map[string]string) bool
|
||||
}
|
||||
|
||||
// String returns the SSE domain as string. For SSE-C the
|
||||
// domain is "SSE-C".
|
||||
func (ssec) String() string { return "SSE-C" }
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
// from the metadata using the SSE-C client key of the HTTP headers
|
||||
// and returns the decrypted object key.
|
||||
func (sse ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
|
||||
clientKey, err := sse.ParseHTTP(h)
|
||||
if err != nil {
|
||||
return
|
||||
// IsRequested returns true and the SSE Type if the HTTP headers
|
||||
// indicate that some form server-side encryption is requested.
|
||||
//
|
||||
// If no SSE headers are present then IsRequested returns false
|
||||
// and no Type.
|
||||
func IsRequested(h http.Header) (Type, bool) {
|
||||
switch {
|
||||
case S3.IsRequested(h):
|
||||
return S3, true
|
||||
case S3KMS.IsRequested(h):
|
||||
return S3KMS, true
|
||||
case SSEC.IsRequested(h):
|
||||
return SSEC, true
|
||||
default:
|
||||
return nil, false
|
||||
}
|
||||
return unsealObjectKey(clientKey, metadata, bucket, object)
|
||||
}
|
||||
|
||||
// UnsealObjectKey extracts and decrypts the sealed object key
|
||||
|
||||
+118
-87
@@ -29,14 +29,12 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/config/heal"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/bucket/lifecycle"
|
||||
"github.com/minio/minio/pkg/bucket/replication"
|
||||
"github.com/minio/minio/pkg/color"
|
||||
"github.com/minio/minio/pkg/console"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
"github.com/minio/minio/pkg/event"
|
||||
"github.com/minio/minio/pkg/hash"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
@@ -45,7 +43,7 @@ import (
|
||||
|
||||
const (
|
||||
dataCrawlSleepPerFolder = time.Millisecond // Time to wait between folders.
|
||||
dataCrawlStartDelay = 5 * time.Minute // Time to wait on startup and between cycles.
|
||||
dataCrawlStartDelay = 1 * time.Minute // Time to wait on startup and between cycles.
|
||||
dataUsageUpdateDirCycles = 16 // Visit all folders every n cycles.
|
||||
|
||||
healDeleteDangling = true
|
||||
@@ -62,11 +60,9 @@ var (
|
||||
crawlerSleeper = newDynamicSleeper(10, 10*time.Second)
|
||||
)
|
||||
|
||||
// initDataCrawler will start the crawler unless disabled.
|
||||
// initDataCrawler will start the crawler in the background.
|
||||
func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
|
||||
if env.Get(envDataUsageCrawlConf, config.EnableOn) == config.EnableOn {
|
||||
go runDataCrawler(ctx, objAPI)
|
||||
}
|
||||
go runDataCrawler(ctx, objAPI)
|
||||
}
|
||||
|
||||
// runDataCrawler will start a data crawler.
|
||||
@@ -173,10 +169,10 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
t := UTCNow()
|
||||
|
||||
logPrefix := color.Green("data-usage: ")
|
||||
logSuffix := color.Blue(" - %v + %v", basePath, cache.Info.Name)
|
||||
logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
|
||||
if intDataUpdateTracker.debug {
|
||||
defer func() {
|
||||
logger.Info(logPrefix+" Crawl time: %v"+logSuffix, time.Since(t))
|
||||
console.Debugf(logPrefix+" Crawl time: %v %s\n", time.Since(t), logSuffix)
|
||||
}()
|
||||
|
||||
}
|
||||
@@ -204,7 +200,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
if ok {
|
||||
s.disks = objAPI.GetDisksID(cache.Disks...)
|
||||
if len(s.disks) != len(cache.Disks) {
|
||||
logger.Info(logPrefix+"Missing disks, want %d, found %d. Cannot heal."+logSuffix, len(cache.Disks), len(s.disks))
|
||||
console.Debugf(logPrefix+"Missing disks, want %d, found %d. Cannot heal. %s\n", len(cache.Disks), len(s.disks), logSuffix)
|
||||
s.disks = s.disks[:0]
|
||||
}
|
||||
}
|
||||
@@ -219,28 +215,28 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
}
|
||||
if len(cache.Info.BloomFilter) > 0 {
|
||||
s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
|
||||
_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
|
||||
_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
|
||||
s.withFilter = nil
|
||||
}
|
||||
}
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"Start crawling. Bloom filter: %v"+logSuffix, s.withFilter != nil)
|
||||
console.Debugf(logPrefix+"Start crawling. Bloom filter: %v %s\n", s.withFilter != nil, logSuffix)
|
||||
}
|
||||
|
||||
done := ctx.Done()
|
||||
var flattenLevels = 2
|
||||
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"Cycle: %v, Entries: %v"+logSuffix, cache.Info.NextCycle, len(cache.Cache))
|
||||
console.Debugf(logPrefix+"Cycle: %v, Entries: %v %s\n", cache.Info.NextCycle, len(cache.Cache), logSuffix)
|
||||
}
|
||||
|
||||
// Always scan flattenLevels deep. Cache root is level 0.
|
||||
todo := []cachedFolder{{name: cache.Info.Name, objectHealProbDiv: 1}}
|
||||
for i := 0; i < flattenLevels; i++ {
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
|
||||
console.Debugf(logPrefix+"Level %v, scanning %v directories. %s\n", i, len(todo), logSuffix)
|
||||
}
|
||||
select {
|
||||
case <-done:
|
||||
@@ -256,7 +252,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
}
|
||||
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"New folders: %v"+logSuffix, s.newFolders)
|
||||
console.Debugf(logPrefix+"New folders: %v %s\n", s.newFolders, logSuffix)
|
||||
}
|
||||
|
||||
// Add new folders first
|
||||
@@ -272,7 +268,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
continue
|
||||
}
|
||||
if du == nil {
|
||||
logger.Info(logPrefix + "no disk usage provided" + logSuffix)
|
||||
console.Debugln(logPrefix + "no disk usage provided" + logSuffix)
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -285,7 +281,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
}
|
||||
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"Existing folders: %v"+logSuffix, len(s.existingFolders))
|
||||
console.Debugf(logPrefix+"Existing folders: %v %s\n", len(s.existingFolders), logSuffix)
|
||||
}
|
||||
|
||||
// Do selective scanning of existing folders.
|
||||
@@ -312,13 +308,13 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
if !s.withFilter.containsDir(folder.name) {
|
||||
if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) {
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
|
||||
console.Debugf(logPrefix+"Skipping non-updated folder: %v %s\n", folder, logSuffix)
|
||||
}
|
||||
s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
|
||||
continue
|
||||
} else {
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"Adding non-updated folder to heal check: %v"+logSuffix, folder.name)
|
||||
console.Debugf(logPrefix+"Adding non-updated folder to heal check: %v %s\n", folder.name, logSuffix)
|
||||
}
|
||||
// Update probability of including objects
|
||||
folder.objectHealProbDiv = s.healFolderInclude
|
||||
@@ -340,7 +336,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
s.newCache.replaceHashed(h, folder.parent, *du)
|
||||
}
|
||||
if s.dataUsageCrawlDebug {
|
||||
logger.Info(logPrefix+"Finished crawl, %v entries"+logSuffix, len(s.newCache.Cache))
|
||||
console.Debugf(logPrefix+"Finished crawl, %v entries %s\n", len(s.newCache.Cache), logSuffix)
|
||||
}
|
||||
s.newCache.Info.LastUpdate = UTCNow()
|
||||
s.newCache.Info.NextCycle++
|
||||
@@ -354,6 +350,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
|
||||
func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFolder, final bool) ([]cachedFolder, error) {
|
||||
var nextFolders []cachedFolder
|
||||
done := ctx.Done()
|
||||
scannerLogPrefix := color.Green("folder-scanner:")
|
||||
for _, folder := range folders {
|
||||
select {
|
||||
case <-done:
|
||||
@@ -369,7 +366,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
var activeLifeCycle *lifecycle.Lifecycle
|
||||
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
|
||||
console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
|
||||
}
|
||||
activeLifeCycle = f.oldCache.Info.lifeCycle
|
||||
filter = nil
|
||||
@@ -380,12 +377,12 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
if !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
|
||||
f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("folder-scanner:")+" Skipping non-updated folder: %v", folder.name)
|
||||
console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
|
||||
}
|
||||
continue
|
||||
} else {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("folder-scanner:")+" Adding non-updated folder to heal check: %v", folder.name)
|
||||
console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
|
||||
}
|
||||
// If probability was already crawlerHealFolderInclude, keep it.
|
||||
folder.objectHealProbDiv = f.healFolderInclude
|
||||
@@ -402,14 +399,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
|
||||
if bucket == "" {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("folder-scanner:")+" no bucket (%s,%s)", f.root, entName)
|
||||
console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
if isReservedOrInvalidBucket(bucket, false) {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("folder-scanner:")+" invalid bucket: %v, entry: %v", bucket, entName)
|
||||
console.Debugf(scannerLogPrefix+" invalid bucket: %v, entry: %v\n", bucket, entName)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -426,8 +423,11 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
cache.addChildString(entName)
|
||||
|
||||
this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
|
||||
delete(existing, h.Key())
|
||||
|
||||
delete(existing, h.Key()) // h.Key() already accounted for.
|
||||
|
||||
cache.addChild(h)
|
||||
|
||||
if final {
|
||||
if exists {
|
||||
f.existingFolders = append(f.existingFolders, this)
|
||||
@@ -454,17 +454,30 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
lifeCycle: activeLifeCycle,
|
||||
heal: thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
|
||||
}
|
||||
sizeSummary, err := f.getSize(item)
|
||||
|
||||
wait()
|
||||
sizeSummary, err := f.getSize(item)
|
||||
if err == errSkipFile {
|
||||
wait() // wait to proceed to next entry.
|
||||
|
||||
return nil
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
|
||||
// successfully read means we have a valid object.
|
||||
|
||||
// Remove filename i.e is the meta file to construct object name
|
||||
item.transformMetaDir()
|
||||
|
||||
// Object already accounted for, remove from heal map,
|
||||
// simply because getSize() function already heals the
|
||||
// object.
|
||||
delete(existing, path.Join(item.bucket, item.objectPath()))
|
||||
|
||||
cache.addSizes(sizeSummary)
|
||||
cache.Objects++
|
||||
cache.ObjSizes.add(sizeSummary.totalSize)
|
||||
|
||||
wait() // wait to proceed to next entry.
|
||||
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
@@ -505,10 +518,11 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
bucket: "",
|
||||
}
|
||||
|
||||
healObjectsPrefix := color.Green("healObjects:")
|
||||
for k := range existing {
|
||||
bucket, prefix := path2BucketObject(k)
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix)
|
||||
console.Debugf(scannerLogPrefix+" checking disappeared folder: %v/%v\n", bucket, prefix)
|
||||
}
|
||||
|
||||
// Dynamic time delay.
|
||||
@@ -516,7 +530,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
resolver.bucket = bucket
|
||||
|
||||
foundObjs := false
|
||||
dangling := true
|
||||
dangling := false
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
err := listPathRaw(ctx, listPathRawOptions{
|
||||
disks: f.disks,
|
||||
@@ -528,17 +542,18 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
// Weird, maybe transient error.
|
||||
agreed: func(entry metaCacheEntry) {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("healObjects:")+" got agreement: %v", entry.name)
|
||||
}
|
||||
if entry.isObject() {
|
||||
dangling = false
|
||||
console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)
|
||||
}
|
||||
},
|
||||
// Some disks have data for this.
|
||||
partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("healObjects:")+" got partial, %d agreed, errs: %v", nAgreed, errs)
|
||||
console.Debugf(healObjectsPrefix+" got partial, %d agreed, errs: %v\n", nAgreed, errs)
|
||||
}
|
||||
|
||||
// agreed value less than expected quorum
|
||||
dangling = nAgreed < resolver.objQuorum || nAgreed < resolver.dirQuorum
|
||||
|
||||
// Sleep and reset.
|
||||
wait()
|
||||
wait = crawlerSleeper.Timer(ctx)
|
||||
@@ -546,8 +561,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
if !ok {
|
||||
for _, err := range errs {
|
||||
if err != nil {
|
||||
// Not all disks are ready, do nothing for now.
|
||||
dangling = false
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -557,12 +570,12 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
}
|
||||
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("healObjects:")+" resolved to: %v, dir: %v", entry.name, entry.isDir())
|
||||
console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir())
|
||||
}
|
||||
|
||||
if entry.isDir() {
|
||||
return
|
||||
}
|
||||
dangling = false
|
||||
// We got an entry which we should be able to heal.
|
||||
fiv, err := entry.fileInfoVersions(bucket)
|
||||
if err != nil {
|
||||
@@ -571,7 +584,9 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
object: entry.name,
|
||||
versionID: "",
|
||||
}, madmin.HealItemObject)
|
||||
logger.LogIf(ctx, err)
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
foundObjs = foundObjs || err == nil
|
||||
return
|
||||
}
|
||||
@@ -584,31 +599,35 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
|
||||
object: fiv.Name,
|
||||
versionID: ver.VersionID,
|
||||
}, madmin.HealItemObject)
|
||||
logger.LogIf(ctx, err)
|
||||
if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
foundObjs = foundObjs || err == nil
|
||||
}
|
||||
},
|
||||
// Too many disks failed.
|
||||
finished: func(errs []error) {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("healObjects:")+" too many errors: %v", errs)
|
||||
console.Debugf(healObjectsPrefix+" too many errors: %v\n", errs)
|
||||
}
|
||||
dangling = false
|
||||
cancel()
|
||||
},
|
||||
})
|
||||
|
||||
if f.dataUsageCrawlDebug && err != nil && err != errFileNotFound {
|
||||
logger.Info(color.Green("healObjects:")+" checking returned value %v (%T)", err, err)
|
||||
console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err)
|
||||
}
|
||||
|
||||
// If we found one or more disks with this folder, delete it.
|
||||
if err == nil && dangling {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("healObjects:")+" deleting dangling directory %s", prefix)
|
||||
console.Debugf(healObjectsPrefix+" deleting dangling directory %s\n", prefix)
|
||||
}
|
||||
// If we have quorum, found directories, but no objects, issue heal to delete the dangling.
|
||||
objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: true},
|
||||
|
||||
objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
|
||||
Recursive: true,
|
||||
Remove: true,
|
||||
},
|
||||
func(bucket, object, versionID string) error {
|
||||
// Wait for each heal as per crawler frequency.
|
||||
wait()
|
||||
@@ -648,6 +667,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
|
||||
var addDir func(entName string, typ os.FileMode) error
|
||||
var dirStack = []string{f.root, folder.name}
|
||||
|
||||
deepScannerLogPrefix := color.Green("deep-scanner:")
|
||||
addDir = func(entName string, typ os.FileMode) error {
|
||||
select {
|
||||
case <-done:
|
||||
@@ -675,7 +695,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
|
||||
var activeLifeCycle *lifecycle.Lifecycle
|
||||
if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
|
||||
if f.dataUsageCrawlDebug {
|
||||
logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
|
||||
console.Debugf(deepScannerLogPrefix+" Prefix %q has active rules\n", prefix)
|
||||
}
|
||||
activeLifeCycle = f.oldCache.Info.lifeCycle
|
||||
}
|
||||
@@ -692,16 +712,21 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
|
||||
heal: hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
|
||||
})
|
||||
|
||||
// Wait to throttle IO
|
||||
wait()
|
||||
|
||||
if err == errSkipFile {
|
||||
// Wait to throttle IO
|
||||
wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
logger.LogIf(ctx, err)
|
||||
cache.addSizes(sizeSummary)
|
||||
cache.Objects++
|
||||
cache.ObjSizes.add(sizeSummary.totalSize)
|
||||
|
||||
// Wait to throttle IO
|
||||
wait()
|
||||
|
||||
return nil
|
||||
}
|
||||
err := readDirFn(path.Join(dirStack...), addDir)
|
||||
@@ -751,6 +776,7 @@ type actionMeta struct {
|
||||
oi ObjectInfo
|
||||
successorModTime time.Time // The modtime of the successor version
|
||||
numVersions int // The number of versions of this object
|
||||
bitRotScan bool // indicates if bitrot check was requested.
|
||||
}
|
||||
|
||||
// applyActions will apply lifecycle checks on to a scanned item.
|
||||
@@ -762,15 +788,20 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
if i.debug {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
applyActionsLogPrefix := color.Green("applyActions:")
|
||||
if i.heal {
|
||||
if i.debug {
|
||||
if meta.oi.VersionID != "" {
|
||||
logger.Info(color.Green("applyActions:")+" heal checking: %v/%v v%s", i.bucket, i.objectPath(), meta.oi.VersionID)
|
||||
console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), meta.oi.VersionID)
|
||||
} else {
|
||||
logger.Info(color.Green("applyActions:")+" heal checking: %v/%v", i.bucket, i.objectPath())
|
||||
console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
|
||||
}
|
||||
}
|
||||
res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling})
|
||||
healOpts := madmin.HealOpts{Remove: healDeleteDangling}
|
||||
if meta.bitRotScan {
|
||||
healOpts.ScanMode = madmin.HealDeepScan
|
||||
}
|
||||
res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, healOpts)
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
return 0
|
||||
}
|
||||
@@ -782,7 +813,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
}
|
||||
if i.lifeCycle == nil {
|
||||
if i.debug {
|
||||
logger.Info(color.Green("applyActions:")+" no lifecycle rules to apply: %q", i.objectPath())
|
||||
console.Debugf(applyActionsLogPrefix+" no lifecycle rules to apply: %q\n", i.objectPath())
|
||||
}
|
||||
return size
|
||||
}
|
||||
@@ -804,9 +835,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
})
|
||||
if i.debug {
|
||||
if versionID != "" {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: %q (version-id=%s), Initial scan: %v", i.objectPath(), versionID, action)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, action)
|
||||
} else {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: %q Initial scan: %v", i.objectPath(), action)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
|
||||
}
|
||||
}
|
||||
switch action {
|
||||
@@ -816,7 +847,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
default:
|
||||
// No action.
|
||||
if i.debug {
|
||||
logger.Info(color.Green("applyActions:")+" object not expirable: %q", i.objectPath())
|
||||
console.Debugf(applyActionsLogPrefix+" object not expirable: %q\n", i.objectPath())
|
||||
}
|
||||
return size
|
||||
}
|
||||
@@ -859,7 +890,7 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
}
|
||||
action = i.lifeCycle.ComputeAction(lcOpts)
|
||||
if i.debug {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: Secondary scan: %v", action)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", action)
|
||||
}
|
||||
switch action {
|
||||
case lifecycle.DeleteAction, lifecycle.DeleteVersionAction:
|
||||
@@ -882,9 +913,9 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
|
||||
if locked {
|
||||
if i.debug {
|
||||
if obj.VersionID != "" {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: %s v%s is locked, not deleting", i.objectPath(), obj.VersionID)
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", i.objectPath(), obj.VersionID)
|
||||
} else {
|
||||
logger.Info(color.Green("applyActions:")+" lifecycle: %s is locked, not deleting", i.objectPath())
|
||||
console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", i.objectPath())
|
||||
}
|
||||
}
|
||||
return size
|
||||
@@ -945,52 +976,52 @@ func (i *crawlItem) objectPath() string {
|
||||
}
|
||||
|
||||
// healReplication will heal a scanned item that has failed replication.
|
||||
func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, meta actionMeta, sizeS *sizeSummary) {
|
||||
if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() {
|
||||
func (i *crawlItem) healReplication(ctx context.Context, o ObjectLayer, oi ObjectInfo, sizeS *sizeSummary) {
|
||||
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
|
||||
// heal delete marker replication failure or versioned delete replication failure
|
||||
if meta.oi.ReplicationStatus == replication.Pending ||
|
||||
meta.oi.ReplicationStatus == replication.Failed ||
|
||||
meta.oi.VersionPurgeStatus == Failed || meta.oi.VersionPurgeStatus == Pending {
|
||||
i.healReplicationDeletes(ctx, o, meta)
|
||||
if oi.ReplicationStatus == replication.Pending ||
|
||||
oi.ReplicationStatus == replication.Failed ||
|
||||
oi.VersionPurgeStatus == Failed || oi.VersionPurgeStatus == Pending {
|
||||
i.healReplicationDeletes(ctx, o, oi)
|
||||
return
|
||||
}
|
||||
}
|
||||
switch meta.oi.ReplicationStatus {
|
||||
switch oi.ReplicationStatus {
|
||||
case replication.Pending:
|
||||
sizeS.pendingSize += meta.oi.Size
|
||||
globalReplicationState.queueReplicaTask(meta.oi)
|
||||
sizeS.pendingSize += oi.Size
|
||||
globalReplicationState.queueReplicaTask(oi)
|
||||
case replication.Failed:
|
||||
sizeS.failedSize += meta.oi.Size
|
||||
globalReplicationState.queueReplicaTask(meta.oi)
|
||||
case replication.Complete:
|
||||
sizeS.replicatedSize += meta.oi.Size
|
||||
sizeS.failedSize += oi.Size
|
||||
globalReplicationState.queueReplicaTask(oi)
|
||||
case replication.Completed, "COMPLETE":
|
||||
sizeS.replicatedSize += oi.Size
|
||||
case replication.Replica:
|
||||
sizeS.replicaSize += meta.oi.Size
|
||||
sizeS.replicaSize += oi.Size
|
||||
}
|
||||
}
|
||||
|
||||
// healReplicationDeletes will heal a scanned deleted item that failed to replicate deletes.
|
||||
func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, meta actionMeta) {
|
||||
func (i *crawlItem) healReplicationDeletes(ctx context.Context, o ObjectLayer, oi ObjectInfo) {
|
||||
// handle soft delete and permanent delete failures here.
|
||||
if meta.oi.DeleteMarker || !meta.oi.VersionPurgeStatus.Empty() {
|
||||
if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
|
||||
versionID := ""
|
||||
dmVersionID := ""
|
||||
if meta.oi.VersionPurgeStatus.Empty() {
|
||||
dmVersionID = meta.oi.VersionID
|
||||
if oi.VersionPurgeStatus.Empty() {
|
||||
dmVersionID = oi.VersionID
|
||||
} else {
|
||||
versionID = meta.oi.VersionID
|
||||
versionID = oi.VersionID
|
||||
}
|
||||
globalReplicationState.queueReplicaDeleteTask(DeletedObjectVersionInfo{
|
||||
DeletedObject: DeletedObject{
|
||||
ObjectName: meta.oi.Name,
|
||||
ObjectName: oi.Name,
|
||||
DeleteMarkerVersionID: dmVersionID,
|
||||
VersionID: versionID,
|
||||
DeleteMarkerReplicationStatus: string(meta.oi.ReplicationStatus),
|
||||
DeleteMarkerMTime: DeleteMarkerMTime{meta.oi.ModTime},
|
||||
DeleteMarker: meta.oi.DeleteMarker,
|
||||
VersionPurgeStatus: meta.oi.VersionPurgeStatus,
|
||||
DeleteMarkerReplicationStatus: string(oi.ReplicationStatus),
|
||||
DeleteMarkerMTime: DeleteMarkerMTime{oi.ModTime},
|
||||
DeleteMarker: oi.DeleteMarker,
|
||||
VersionPurgeStatus: oi.VersionPurgeStatus,
|
||||
},
|
||||
Bucket: meta.oi.Bucket,
|
||||
Bucket: oi.Bucket,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+15
-12
@@ -34,6 +34,7 @@ import (
|
||||
"github.com/minio/minio/cmd/config"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/color"
|
||||
"github.com/minio/minio/pkg/console"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
"github.com/willf/bloom"
|
||||
)
|
||||
@@ -171,16 +172,17 @@ func (d *dataUpdateTracker) current() uint64 {
|
||||
// latestWithDir returns the highest index that contains the directory.
|
||||
// This means that any cycle higher than this does NOT contain the entry.
|
||||
func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
|
||||
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
|
||||
bucket, _ := path2BucketObjectWithBasePath("", dir)
|
||||
if bucket == "" {
|
||||
if d.debug && len(dir) > 0 {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", dir)
|
||||
console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
|
||||
}
|
||||
return d.current()
|
||||
}
|
||||
if isReservedOrInvalidBucket(bucket, false) {
|
||||
if d.debug {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, dir)
|
||||
console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, dir)
|
||||
}
|
||||
return d.current()
|
||||
}
|
||||
@@ -191,7 +193,7 @@ func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
|
||||
return d.Current.idx
|
||||
}
|
||||
if d.debug {
|
||||
logger.Info("current bloom does NOT contains dir %s", dir)
|
||||
console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
|
||||
}
|
||||
|
||||
idx := d.Current.idx - 1
|
||||
@@ -281,7 +283,7 @@ func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Durati
|
||||
d.Saved = UTCNow()
|
||||
err := d.serialize(&buf)
|
||||
if d.debug {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v", buf.Len(), d.Current.idx)
|
||||
console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
|
||||
}
|
||||
d.dirty = false
|
||||
d.mu.Unlock()
|
||||
@@ -404,7 +406,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
|
||||
}
|
||||
switch tmp[0] {
|
||||
case 1, 2, 3:
|
||||
logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
|
||||
console.Println(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
|
||||
return nil
|
||||
case dataUpdateTrackerVersion:
|
||||
default:
|
||||
@@ -481,14 +483,14 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
|
||||
bucket, _ := path2BucketObjectWithBasePath("", in)
|
||||
if bucket == "" {
|
||||
if d.debug && len(in) > 0 {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
|
||||
console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
if isReservedOrInvalidBucket(bucket, false) {
|
||||
if d.debug {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
|
||||
console.Debugf(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
|
||||
}
|
||||
continue
|
||||
}
|
||||
@@ -498,7 +500,7 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
|
||||
d.mu.Lock()
|
||||
for i := range split {
|
||||
if d.debug {
|
||||
logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
|
||||
console.Debugln(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
|
||||
}
|
||||
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
|
||||
}
|
||||
@@ -510,16 +512,17 @@ func (d *dataUpdateTracker) startCollector(ctx context.Context) {
|
||||
// markDirty adds the supplied path to the current bloom filter.
|
||||
func (d *dataUpdateTracker) markDirty(in string) {
|
||||
bucket, _ := path2BucketObjectWithBasePath("", in)
|
||||
dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
|
||||
if bucket == "" {
|
||||
if d.debug && len(in) > 0 {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
|
||||
console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", in)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if isReservedOrInvalidBucket(bucket, false) {
|
||||
if d.debug && false {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
|
||||
console.Debugf(dateUpdateTrackerLogPrefix+" isReservedOrInvalidBucket: %v, entry: %v\n", bucket, in)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -529,7 +532,7 @@ func (d *dataUpdateTracker) markDirty(in string) {
|
||||
d.mu.Lock()
|
||||
for i := range split {
|
||||
if d.debug {
|
||||
logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
|
||||
console.Debugln(dateUpdateTrackerLogPrefix + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
|
||||
}
|
||||
d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
|
||||
}
|
||||
@@ -620,7 +623,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequ
|
||||
if d.Current.idx != current {
|
||||
d.dirty = true
|
||||
if d.debug {
|
||||
logger.Info(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v", d.Current.idx, current)
|
||||
console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
|
||||
}
|
||||
|
||||
d.History = append(d.History, d.Current)
|
||||
|
||||
@@ -225,7 +225,7 @@ func TestDataUpdateTracker(t *testing.T) {
|
||||
|
||||
// Rerun test with returned bfr2
|
||||
bf := dut.newBloomFilter()
|
||||
_, err = bf.ReadFrom(bytes.NewBuffer(bfr2.Filter))
|
||||
_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
+64
-22
@@ -17,11 +17,11 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -56,13 +56,29 @@ type dataUsageEntry struct {
|
||||
Children dataUsageHashMap
|
||||
}
|
||||
|
||||
// dataUsageCache contains a cache of data usage entries.
|
||||
//msgp:tuple dataUsageEntryV2
|
||||
type dataUsageEntryV2 struct {
|
||||
// These fields do no include any children.
|
||||
Size int64
|
||||
Objects uint64
|
||||
ObjSizes sizeHistogram
|
||||
Children dataUsageHashMap
|
||||
}
|
||||
|
||||
// dataUsageCache contains a cache of data usage entries latest version 3.
|
||||
type dataUsageCache struct {
|
||||
Info dataUsageCacheInfo
|
||||
Disks []string
|
||||
Cache map[string]dataUsageEntry
|
||||
}
|
||||
|
||||
// dataUsageCache contains a cache of data usage entries version 2.
|
||||
type dataUsageCacheV2 struct {
|
||||
Info dataUsageCacheInfo
|
||||
Disks []string
|
||||
Cache map[string]dataUsageEntryV2
|
||||
}
|
||||
|
||||
//msgp:ignore dataUsageEntryInfo
|
||||
type dataUsageEntryInfo struct {
|
||||
Name string
|
||||
@@ -458,7 +474,7 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
|
||||
}
|
||||
|
||||
type objectIO interface {
|
||||
GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error)
|
||||
GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error)
|
||||
PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
|
||||
}
|
||||
|
||||
@@ -466,8 +482,7 @@ type objectIO interface {
|
||||
// Only backend errors are returned as errors.
|
||||
// If the object is not found or unable to deserialize d is cleared and nil error is returned.
|
||||
func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
|
||||
var buf bytes.Buffer
|
||||
err := store.GetObject(ctx, dataUsageBucket, name, 0, -1, &buf, "", ObjectOptions{})
|
||||
r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, readLock, ObjectOptions{})
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case ObjectNotFound:
|
||||
@@ -479,10 +494,10 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
|
||||
*d = dataUsageCache{}
|
||||
return nil
|
||||
}
|
||||
err = d.deserialize(&buf)
|
||||
if err != nil {
|
||||
defer r.Close()
|
||||
if err := d.deserialize(r); err != nil {
|
||||
*d = dataUsageCache{}
|
||||
logger.LogIf(ctx, err)
|
||||
logger.LogOnceIf(ctx, err, err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -494,6 +509,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
pw.CloseWithError(d.serializeTo(pw))
|
||||
}()
|
||||
defer pr.Close()
|
||||
|
||||
r, err := hash.NewReader(pr, -1, "", "", -1, false)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -513,12 +529,16 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
// dataUsageCacheVer indicates the cache version.
|
||||
// Bumping the cache version will drop data from previous versions
|
||||
// and write new data with the new version.
|
||||
const dataUsageCacheVer = 3
|
||||
const (
|
||||
dataUsageCacheVerV3 = 3
|
||||
dataUsageCacheVerV2 = 2
|
||||
dataUsageCacheVerV1 = 1
|
||||
)
|
||||
|
||||
// serialize the contents of the cache.
|
||||
func (d *dataUsageCache) serializeTo(dst io.Writer) error {
|
||||
// Add version and compress.
|
||||
_, err := dst.Write([]byte{dataUsageCacheVer})
|
||||
_, err := dst.Write([]byte{dataUsageCacheVerV3})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -553,21 +573,43 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
switch b[0] {
|
||||
case 1, 2:
|
||||
case dataUsageCacheVerV1:
|
||||
return errors.New("cache version deprecated (will autoupdate)")
|
||||
case dataUsageCacheVer:
|
||||
default:
|
||||
return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
|
||||
}
|
||||
case dataUsageCacheVerV2:
|
||||
// Zstd compressed.
|
||||
dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dec.Close()
|
||||
|
||||
// Zstd compressed.
|
||||
dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dec.Close()
|
||||
dold := &dataUsageCacheV2{}
|
||||
if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil {
|
||||
return err
|
||||
}
|
||||
d.Info = dold.Info
|
||||
d.Disks = dold.Disks
|
||||
d.Cache = make(map[string]dataUsageEntry, len(dold.Cache))
|
||||
for k, v := range dold.Cache {
|
||||
d.Cache[k] = dataUsageEntry{
|
||||
Size: v.Size,
|
||||
Objects: v.Objects,
|
||||
ObjSizes: v.ObjSizes,
|
||||
Children: v.Children,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
case dataUsageCacheVerV3:
|
||||
// Zstd compressed.
|
||||
dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer dec.Close()
|
||||
|
||||
return d.DecodeMsg(msgp.NewReader(dec))
|
||||
return d.DecodeMsg(msgp.NewReader(dec))
|
||||
}
|
||||
return fmt.Errorf("dataUsageCache: unknown version: %d", int(b[0]))
|
||||
}
|
||||
|
||||
// Trim this from start+end of hashes.
|
||||
|
||||
@@ -484,6 +484,277 @@ func (z *dataUsageCacheInfo) Msgsize() (s int) {
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *dataUsageCacheV2) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "Info":
|
||||
err = z.Info.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Info")
|
||||
return
|
||||
}
|
||||
case "Disks":
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Disks")
|
||||
return
|
||||
}
|
||||
if cap(z.Disks) >= int(zb0002) {
|
||||
z.Disks = (z.Disks)[:zb0002]
|
||||
} else {
|
||||
z.Disks = make([]string, zb0002)
|
||||
}
|
||||
for za0001 := range z.Disks {
|
||||
z.Disks[za0001], err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Disks", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Cache":
|
||||
var zb0003 uint32
|
||||
zb0003, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache")
|
||||
return
|
||||
}
|
||||
if z.Cache == nil {
|
||||
z.Cache = make(map[string]dataUsageEntryV2, zb0003)
|
||||
} else if len(z.Cache) > 0 {
|
||||
for key := range z.Cache {
|
||||
delete(z.Cache, key)
|
||||
}
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
zb0003--
|
||||
var za0002 string
|
||||
var za0003 dataUsageEntryV2
|
||||
za0002, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache")
|
||||
return
|
||||
}
|
||||
err = za0003.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache", za0002)
|
||||
return
|
||||
}
|
||||
z.Cache[za0002] = za0003
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *dataUsageCacheV2) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 3
|
||||
// write "Info"
|
||||
err = en.Append(0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = z.Info.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Info")
|
||||
return
|
||||
}
|
||||
// write "Disks"
|
||||
err = en.Append(0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(len(z.Disks)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Disks")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.Disks {
|
||||
err = en.WriteString(z.Disks[za0001])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Disks", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "Cache"
|
||||
err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteMapHeader(uint32(len(z.Cache)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache")
|
||||
return
|
||||
}
|
||||
for za0002, za0003 := range z.Cache {
|
||||
err = en.WriteString(za0002)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache")
|
||||
return
|
||||
}
|
||||
err = za0003.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *dataUsageCacheV2) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 3
|
||||
// string "Info"
|
||||
o = append(o, 0x83, 0xa4, 0x49, 0x6e, 0x66, 0x6f)
|
||||
o, err = z.Info.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Info")
|
||||
return
|
||||
}
|
||||
// string "Disks"
|
||||
o = append(o, 0xa5, 0x44, 0x69, 0x73, 0x6b, 0x73)
|
||||
o = msgp.AppendArrayHeader(o, uint32(len(z.Disks)))
|
||||
for za0001 := range z.Disks {
|
||||
o = msgp.AppendString(o, z.Disks[za0001])
|
||||
}
|
||||
// string "Cache"
|
||||
o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65)
|
||||
o = msgp.AppendMapHeader(o, uint32(len(z.Cache)))
|
||||
for za0002, za0003 := range z.Cache {
|
||||
o = msgp.AppendString(o, za0002)
|
||||
o, err = za0003.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *dataUsageCacheV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "Info":
|
||||
bts, err = z.Info.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Info")
|
||||
return
|
||||
}
|
||||
case "Disks":
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Disks")
|
||||
return
|
||||
}
|
||||
if cap(z.Disks) >= int(zb0002) {
|
||||
z.Disks = (z.Disks)[:zb0002]
|
||||
} else {
|
||||
z.Disks = make([]string, zb0002)
|
||||
}
|
||||
for za0001 := range z.Disks {
|
||||
z.Disks[za0001], bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Disks", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Cache":
|
||||
var zb0003 uint32
|
||||
zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache")
|
||||
return
|
||||
}
|
||||
if z.Cache == nil {
|
||||
z.Cache = make(map[string]dataUsageEntryV2, zb0003)
|
||||
} else if len(z.Cache) > 0 {
|
||||
for key := range z.Cache {
|
||||
delete(z.Cache, key)
|
||||
}
|
||||
}
|
||||
for zb0003 > 0 {
|
||||
var za0002 string
|
||||
var za0003 dataUsageEntryV2
|
||||
zb0003--
|
||||
za0002, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache")
|
||||
return
|
||||
}
|
||||
bts, err = za0003.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Cache", za0002)
|
||||
return
|
||||
}
|
||||
z.Cache[za0002] = za0003
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *dataUsageCacheV2) Msgsize() (s int) {
|
||||
s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.ArrayHeaderSize
|
||||
for za0001 := range z.Disks {
|
||||
s += msgp.StringPrefixSize + len(z.Disks[za0001])
|
||||
}
|
||||
s += 6 + msgp.MapHeaderSize
|
||||
if z.Cache != nil {
|
||||
for za0002, za0003 := range z.Cache {
|
||||
_ = za0003
|
||||
s += msgp.StringPrefixSize + len(za0002) + za0003.Msgsize()
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var zb0001 uint32
|
||||
@@ -705,6 +976,163 @@ func (z *dataUsageEntry) Msgsize() (s int) {
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *dataUsageEntryV2) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != 4 {
|
||||
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
|
||||
return
|
||||
}
|
||||
z.Size, err = dc.ReadInt64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
z.Objects, err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Objects")
|
||||
return
|
||||
}
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes")
|
||||
return
|
||||
}
|
||||
if zb0002 != uint32(dataUsageBucketLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
|
||||
return
|
||||
}
|
||||
for za0001 := range z.ObjSizes {
|
||||
z.ObjSizes[za0001], err = dc.ReadUint64()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
err = z.Children.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *dataUsageEntryV2) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// array header, size 4
|
||||
err = en.Append(0x94)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt64(z.Size)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
err = en.WriteUint64(z.Objects)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Objects")
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(dataUsageBucketLen))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.ObjSizes {
|
||||
err = en.WriteUint64(z.ObjSizes[za0001])
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
err = z.Children.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *dataUsageEntryV2) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// array header, size 4
|
||||
o = append(o, 0x94)
|
||||
o = msgp.AppendInt64(o, z.Size)
|
||||
o = msgp.AppendUint64(o, z.Objects)
|
||||
o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
|
||||
for za0001 := range z.ObjSizes {
|
||||
o = msgp.AppendUint64(o, z.ObjSizes[za0001])
|
||||
}
|
||||
o, err = z.Children.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *dataUsageEntryV2) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
if zb0001 != 4 {
|
||||
err = msgp.ArrayError{Wanted: 4, Got: zb0001}
|
||||
return
|
||||
}
|
||||
z.Size, bts, err = msgp.ReadInt64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
z.Objects, bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Objects")
|
||||
return
|
||||
}
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes")
|
||||
return
|
||||
}
|
||||
if zb0002 != uint32(dataUsageBucketLen) {
|
||||
err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002}
|
||||
return
|
||||
}
|
||||
for za0001 := range z.ObjSizes {
|
||||
z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "ObjSizes", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
bts, err = z.Children.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Children")
|
||||
return
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *dataUsageEntryV2) Msgsize() (s int) {
|
||||
s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize()
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
{
|
||||
|
||||
@@ -235,6 +235,119 @@ func BenchmarkDecodedataUsageCacheInfo(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshaldataUsageCacheV2(t *testing.T) {
|
||||
v := dataUsageCacheV2{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgdataUsageCacheV2(b *testing.B) {
|
||||
v := dataUsageCacheV2{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgdataUsageCacheV2(b *testing.B) {
|
||||
v := dataUsageCacheV2{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshaldataUsageCacheV2(b *testing.B) {
|
||||
v := dataUsageCacheV2{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodedataUsageCacheV2(t *testing.T) {
|
||||
v := dataUsageCacheV2{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodedataUsageCacheV2 Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := dataUsageCacheV2{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodedataUsageCacheV2(b *testing.B) {
|
||||
v := dataUsageCacheV2{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodedataUsageCacheV2(b *testing.B) {
|
||||
v := dataUsageCacheV2{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshaldataUsageEntry(t *testing.T) {
|
||||
v := dataUsageEntry{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
@@ -348,6 +461,119 @@ func BenchmarkDecodedataUsageEntry(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshaldataUsageEntryV2(t *testing.T) {
|
||||
v := dataUsageEntryV2{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgdataUsageEntryV2(b *testing.B) {
|
||||
v := dataUsageEntryV2{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgdataUsageEntryV2(b *testing.B) {
|
||||
v := dataUsageEntryV2{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshaldataUsageEntryV2(b *testing.B) {
|
||||
v := dataUsageEntryV2{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodedataUsageEntryV2(t *testing.T) {
|
||||
v := dataUsageEntryV2{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodedataUsageEntryV2 Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := dataUsageEntryV2{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodedataUsageEntryV2(b *testing.B) {
|
||||
v := dataUsageEntryV2{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodedataUsageEntryV2(b *testing.B) {
|
||||
v := dataUsageEntryV2{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalsizeHistogram(t *testing.T) {
|
||||
v := sizeHistogram{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
|
||||
@@ -27,7 +27,6 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
envDataUsageCrawlConf = "MINIO_DISK_USAGE_CRAWL_ENABLE"
|
||||
envDataUsageCrawlDebug = "MINIO_DISK_USAGE_CRAWL_DEBUG"
|
||||
|
||||
dataUsageRoot = SlashSeparator
|
||||
|
||||
@@ -962,7 +962,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
|
||||
}
|
||||
if globalCacheKMS != nil {
|
||||
// clean up internal SSE cache metadata
|
||||
delete(gr.ObjInfo.UserDefined, crypto.SSEHeader)
|
||||
delete(gr.ObjInfo.UserDefined, xhttp.AmzServerSideEncryption)
|
||||
}
|
||||
if !rngInfo.Empty() {
|
||||
// overlay Size with actual object size and not the range size
|
||||
|
||||
@@ -163,8 +163,8 @@ func cmpReaders(r1, r2 io.Reader) (bool, string) {
|
||||
|
||||
func TestCmpReaders(t *testing.T) {
|
||||
{
|
||||
r1 := bytes.NewBuffer([]byte("abc"))
|
||||
r2 := bytes.NewBuffer([]byte("abc"))
|
||||
r1 := bytes.NewReader([]byte("abc"))
|
||||
r2 := bytes.NewReader([]byte("abc"))
|
||||
ok, msg := cmpReaders(r1, r2)
|
||||
if !(ok && msg == "") {
|
||||
t.Fatalf("unexpected")
|
||||
@@ -172,8 +172,8 @@ func TestCmpReaders(t *testing.T) {
|
||||
}
|
||||
|
||||
{
|
||||
r1 := bytes.NewBuffer([]byte("abc"))
|
||||
r2 := bytes.NewBuffer([]byte("abcd"))
|
||||
r1 := bytes.NewReader([]byte("abc"))
|
||||
r2 := bytes.NewReader([]byte("abcd"))
|
||||
ok, _ := cmpReaders(r1, r2)
|
||||
if ok {
|
||||
t.Fatalf("unexpected")
|
||||
|
||||
@@ -353,9 +353,7 @@ func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte,
|
||||
|
||||
// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a
|
||||
// reader
|
||||
func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
|
||||
length int64, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (
|
||||
io.Reader, error) {
|
||||
func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) {
|
||||
|
||||
bucket, object := oi.Bucket, oi.Name
|
||||
// Single part case
|
||||
@@ -386,13 +384,13 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, offset,
|
||||
header: h,
|
||||
bucket: bucket,
|
||||
object: object,
|
||||
customerKeyHeader: h.Get(crypto.SSECKey),
|
||||
customerKeyHeader: h.Get(xhttp.AmzServerSideEncryptionCustomerKey),
|
||||
copySource: copySource,
|
||||
metadata: cloneMSS(oi.UserDefined),
|
||||
}
|
||||
|
||||
if w.copySource {
|
||||
w.customerKeyHeader = h.Get(crypto.SSECopyKey)
|
||||
w.customerKeyHeader = h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey)
|
||||
}
|
||||
|
||||
if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
|
||||
@@ -434,12 +432,12 @@ func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
|
||||
var err error
|
||||
if d.copySource {
|
||||
if crypto.SSEC.IsEncrypted(d.metadata) {
|
||||
d.header.Set(crypto.SSECopyKey, d.customerKeyHeader)
|
||||
d.header.Set(xhttp.AmzServerSideEncryptionCopyCustomerKey, d.customerKeyHeader)
|
||||
key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
|
||||
}
|
||||
} else {
|
||||
if crypto.SSEC.IsEncrypted(d.metadata) {
|
||||
d.header.Set(crypto.SSECKey, d.customerKeyHeader)
|
||||
d.header.Set(xhttp.AmzServerSideEncryptionCustomerKey, d.customerKeyHeader)
|
||||
key, err = ParseSSECustomerHeader(d.header)
|
||||
}
|
||||
}
|
||||
|
||||
+51
-50
@@ -29,6 +29,7 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
"github.com/minio/minio/cmd/crypto"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
@@ -38,27 +39,27 @@ var encryptRequestTests = []struct {
|
||||
}{
|
||||
{
|
||||
header: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
crypto.SSECKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256",
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{},
|
||||
},
|
||||
{
|
||||
header: map[string]string{
|
||||
crypto.SSECAlgorithm: "AES256",
|
||||
crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
crypto.SSECKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256",
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==",
|
||||
},
|
||||
metadata: map[string]string{
|
||||
crypto.SSECKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func TestEncryptRequest(t *testing.T) {
|
||||
defer func(flag bool) { globalIsSSL = flag }(globalIsSSL)
|
||||
globalIsSSL = true
|
||||
defer func(flag bool) { globalIsTLS = flag }(globalIsTLS)
|
||||
globalIsTLS = true
|
||||
for i, test := range encryptRequestTests {
|
||||
content := bytes.NewReader(make([]byte, 64))
|
||||
req := &http.Request{Header: http.Header{}}
|
||||
@@ -70,13 +71,13 @@ func TestEncryptRequest(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatalf("Test %d: Failed to encrypt request: %v", i, err)
|
||||
}
|
||||
if kdf, ok := test.metadata[crypto.SSESealAlgorithm]; !ok {
|
||||
if kdf, ok := test.metadata[crypto.MetaAlgorithm]; !ok {
|
||||
t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf)
|
||||
}
|
||||
if iv, ok := test.metadata[crypto.SSEIV]; !ok {
|
||||
if iv, ok := test.metadata[crypto.MetaIV]; !ok {
|
||||
t.Errorf("Test %d: crypto.SSEIV must be part of metadata: %v", i, iv)
|
||||
}
|
||||
if mac, ok := test.metadata[crypto.SSECSealedKey]; !ok {
|
||||
if mac, ok := test.metadata[crypto.MetaSealedKeySSEC]; !ok {
|
||||
t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac)
|
||||
}
|
||||
}
|
||||
@@ -93,33 +94,33 @@ var decryptObjectInfoTests = []struct {
|
||||
expErr: nil,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
|
||||
request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
|
||||
request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}},
|
||||
expErr: nil,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
|
||||
request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
|
||||
info: ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
|
||||
request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}},
|
||||
expErr: nil,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.SSECSealedKey: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaSealedKeySSEC: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}},
|
||||
request: &http.Request{Header: http.Header{}},
|
||||
expErr: errEncryptedObject,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{}},
|
||||
request: &http.Request{Method: http.MethodGet, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
|
||||
request: &http.Request{Method: http.MethodGet, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
|
||||
expErr: errInvalidEncryptionParameters,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 100, UserDefined: map[string]string{}},
|
||||
request: &http.Request{Method: http.MethodHead, Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
|
||||
request: &http.Request{Method: http.MethodHead, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
|
||||
expErr: errInvalidEncryptionParameters,
|
||||
},
|
||||
{
|
||||
info: ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm}},
|
||||
request: &http.Request{Header: http.Header{crypto.SSECAlgorithm: []string{crypto.SSEAlgorithmAES256}}},
|
||||
info: ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}},
|
||||
request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}},
|
||||
expErr: errObjectTampered,
|
||||
},
|
||||
}
|
||||
@@ -221,10 +222,10 @@ func TestGetDecryptedRange_Issue50(t *testing.T) {
|
||||
Name: "object",
|
||||
Size: 595160760,
|
||||
UserDefined: map[string]string{
|
||||
crypto.SSEMultipart: "",
|
||||
crypto.SSEIV: "HTexa=",
|
||||
crypto.SSESealAlgorithm: "DAREv2-HMAC-SHA256",
|
||||
crypto.SSECSealedKey: "IAA8PGAA==",
|
||||
crypto.MetaMultipart: "",
|
||||
crypto.MetaIV: "HTexa=",
|
||||
crypto.MetaAlgorithm: "DAREv2-HMAC-SHA256",
|
||||
crypto.MetaSealedKeySSEC: "IAA8PGAA==",
|
||||
ReservedMetadataPrefix + "actual-size": "594870264",
|
||||
"content-type": "application/octet-stream",
|
||||
"etag": "166b1545b4c1535294ee0686678bea8c-2",
|
||||
@@ -276,11 +277,11 @@ func TestGetDecryptedRange(t *testing.T) {
|
||||
}
|
||||
udMap = func(isMulti bool) map[string]string {
|
||||
m := map[string]string{
|
||||
crypto.SSESealAlgorithm: crypto.InsecureSealAlgorithm,
|
||||
crypto.SSEMultipart: "1",
|
||||
crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm,
|
||||
crypto.MetaMultipart: "1",
|
||||
}
|
||||
if !isMulti {
|
||||
delete(m, crypto.SSEMultipart)
|
||||
delete(m, crypto.MetaMultipart)
|
||||
}
|
||||
return m
|
||||
}
|
||||
@@ -553,56 +554,56 @@ var getDefaultOptsTests = []struct {
|
||||
encryptionType encrypt.Type
|
||||
err error
|
||||
}{
|
||||
{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
|
||||
crypto.SSECKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: encrypt.SSEC,
|
||||
err: nil}, // 0
|
||||
{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
|
||||
crypto.SSECKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: true,
|
||||
metadata: nil,
|
||||
encryptionType: "",
|
||||
err: nil}, // 1
|
||||
{headers: http.Header{crypto.SSECAlgorithm: []string{"AES256"},
|
||||
crypto.SSECKey: []string{"Mz"},
|
||||
crypto.SSECKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
{headers: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"},
|
||||
xhttp.AmzServerSideEncryptionCustomerKey: []string{"Mz"},
|
||||
xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: "",
|
||||
err: crypto.ErrInvalidCustomerKey}, // 2
|
||||
{headers: http.Header{crypto.SSEHeader: []string{"AES256"}},
|
||||
{headers: http.Header{xhttp.AmzServerSideEncryption: []string{"AES256"}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: encrypt.S3,
|
||||
err: nil}, // 3
|
||||
{headers: http.Header{},
|
||||
copySource: false,
|
||||
metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
crypto.S3KMSKeyID: "kms-key",
|
||||
crypto.S3KMSSealedKey: "m-key"},
|
||||
metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
crypto.MetaKeyID: "kms-key",
|
||||
crypto.MetaDataEncryptionKey: "m-key"},
|
||||
encryptionType: encrypt.S3,
|
||||
err: nil}, // 4
|
||||
{headers: http.Header{},
|
||||
copySource: true,
|
||||
metadata: map[string]string{crypto.S3SealedKey: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
crypto.S3KMSKeyID: "kms-key",
|
||||
crypto.S3KMSSealedKey: "m-key"},
|
||||
metadata: map[string]string{crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)),
|
||||
crypto.MetaKeyID: "kms-key",
|
||||
crypto.MetaDataEncryptionKey: "m-key"},
|
||||
encryptionType: "",
|
||||
err: nil}, // 5
|
||||
{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
|
||||
crypto.SSECopyKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: true,
|
||||
metadata: nil,
|
||||
encryptionType: encrypt.SSEC,
|
||||
err: nil}, // 6
|
||||
{headers: http.Header{crypto.SSECopyAlgorithm: []string{"AES256"},
|
||||
crypto.SSECopyKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
crypto.SSECopyKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
{headers: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="},
|
||||
xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}},
|
||||
copySource: false,
|
||||
metadata: nil,
|
||||
encryptionType: "",
|
||||
|
||||
+16
-21
@@ -18,7 +18,6 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
@@ -38,7 +37,6 @@ import (
|
||||
"github.com/minio/minio/cmd/config"
|
||||
xhttp "github.com/minio/minio/cmd/http"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/cmd/rest"
|
||||
"github.com/minio/minio/pkg/env"
|
||||
"github.com/minio/minio/pkg/mountinfo"
|
||||
xnet "github.com/minio/minio/pkg/net"
|
||||
@@ -59,7 +57,7 @@ const (
|
||||
// See proxyRequest() for details.
|
||||
type ProxyEndpoint struct {
|
||||
Endpoint
|
||||
Transport *http.Transport
|
||||
Transport http.RoundTripper
|
||||
}
|
||||
|
||||
// Endpoint - any type of endpoint.
|
||||
@@ -198,7 +196,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ZoneEndpoints represent endpoints in a given zone
|
||||
// ZoneEndpoints represent endpoints in a given pool
|
||||
// along with its setCount and setDriveCount.
|
||||
type ZoneEndpoints struct {
|
||||
SetCount int
|
||||
@@ -209,8 +207,8 @@ type ZoneEndpoints struct {
|
||||
// EndpointServerPools - list of list of endpoints
|
||||
type EndpointServerPools []ZoneEndpoints
|
||||
|
||||
// GetLocalZoneIdx returns the zone which endpoint belongs to locally.
|
||||
// if ep is remote this code will return -1 zoneIndex
|
||||
// GetLocalZoneIdx returns the pool which endpoint belongs to locally.
|
||||
// if ep is remote this code will return -1 poolIndex
|
||||
func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
|
||||
for i, zep := range l {
|
||||
for _, cep := range zep.Endpoints {
|
||||
@@ -224,7 +222,7 @@ func (l EndpointServerPools) GetLocalZoneIdx(ep Endpoint) int {
|
||||
return -1
|
||||
}
|
||||
|
||||
// Add add zone endpoints
|
||||
// Add add pool endpoints
|
||||
func (l *EndpointServerPools) Add(zeps ZoneEndpoints) error {
|
||||
existSet := set.NewStringSet()
|
||||
for _, zep := range *l {
|
||||
@@ -353,6 +351,14 @@ func (endpoints Endpoints) GetString(i int) string {
|
||||
return endpoints[i].String()
|
||||
}
|
||||
|
||||
// GetAllStrings - returns allstring of all endpoints
|
||||
func (endpoints Endpoints) GetAllStrings() (all []string) {
|
||||
for _, e := range endpoints {
|
||||
all = append(all, e.String())
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func hostResolveToLocalhost(endpoint Endpoint) bool {
|
||||
hostIPs, err := getHostIP(endpoint.Hostname())
|
||||
if err != nil {
|
||||
@@ -472,8 +478,8 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
|
||||
// participate atleast one disk and be local.
|
||||
//
|
||||
// In special cases for replica set with expanded
|
||||
// zone setups we need to make sure to provide
|
||||
// value of foundPrevLocal from zone1 if we already
|
||||
// pool setups we need to make sure to provide
|
||||
// value of foundPrevLocal from pool1 if we already
|
||||
// found a local setup. Only if we haven't found
|
||||
// previous local we continue to wait to look for
|
||||
// atleast one local.
|
||||
@@ -873,20 +879,9 @@ func GetProxyEndpoints(endpointServerPools EndpointServerPools) []ProxyEndpoint
|
||||
}
|
||||
proxyEpSet.Add(host)
|
||||
|
||||
var tlsConfig *tls.Config
|
||||
if globalIsSSL {
|
||||
tlsConfig = &tls.Config{
|
||||
ServerName: endpoint.Hostname(),
|
||||
RootCAs: globalRootCAs,
|
||||
}
|
||||
}
|
||||
|
||||
// allow transport to be HTTP/1.1 for proxying.
|
||||
tr := newCustomHTTPProxyTransport(tlsConfig, rest.DefaultTimeout)()
|
||||
|
||||
proxyEps = append(proxyEps, ProxyEndpoint{
|
||||
Endpoint: endpoint,
|
||||
Transport: tr,
|
||||
Transport: globalProxyTransport,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+76
-10
@@ -18,7 +18,8 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/pkg/sync/errgroup"
|
||||
@@ -37,6 +38,65 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
|
||||
return newDisks
|
||||
}
|
||||
|
||||
type sortSlices struct {
|
||||
disks []StorageAPI
|
||||
infos []DiskInfo
|
||||
}
|
||||
|
||||
type sortByOther sortSlices
|
||||
|
||||
func (sbo sortByOther) Len() int {
|
||||
return len(sbo.disks)
|
||||
}
|
||||
|
||||
func (sbo sortByOther) Swap(i, j int) {
|
||||
sbo.disks[i], sbo.disks[j] = sbo.disks[j], sbo.disks[i]
|
||||
sbo.infos[i], sbo.infos[j] = sbo.infos[j], sbo.infos[i]
|
||||
}
|
||||
|
||||
func (sbo sortByOther) Less(i, j int) bool {
|
||||
return sbo.infos[i].UsedInodes < sbo.infos[j].UsedInodes
|
||||
}
|
||||
|
||||
func (er erasureObjects) getOnlineDisksSortedByUsedInodes() (newDisks []StorageAPI) {
|
||||
disks := er.getDisks()
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
var infos []DiskInfo
|
||||
for _, i := range hashOrder(UTCNow().String(), len(disks)) {
|
||||
i := i
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
if disks[i-1] == nil {
|
||||
return
|
||||
}
|
||||
di, err := disks[i-1].DiskInfo(context.Background())
|
||||
if err != nil || di.Healing {
|
||||
|
||||
// - Do not consume disks which are not reachable
|
||||
// unformatted or simply not accessible for some reason.
|
||||
//
|
||||
// - Do not consume disks which are being healed
|
||||
//
|
||||
// - Future: skip busy disks
|
||||
return
|
||||
}
|
||||
|
||||
mu.Lock()
|
||||
newDisks = append(newDisks, disks[i-1])
|
||||
infos = append(infos, di)
|
||||
mu.Unlock()
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
slices := sortSlices{newDisks, infos}
|
||||
sort.Sort(sortByOther(slices))
|
||||
|
||||
return slices.disks
|
||||
}
|
||||
|
||||
func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
|
||||
disks := er.getDisks()
|
||||
var wg sync.WaitGroup
|
||||
@@ -147,24 +207,29 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
|
||||
// object is "a/b/c/d", stat makes sure that objects ""a/b/c""
|
||||
// "a/b" and "a" do not exist.
|
||||
func (er erasureObjects) parentDirIsObject(ctx context.Context, bucket, parent string) bool {
|
||||
var isParentDirObject func(string) bool
|
||||
isParentDirObject = func(p string) bool {
|
||||
if p == "." || p == SlashSeparator {
|
||||
path := ""
|
||||
segments := strings.Split(parent, slashSeparator)
|
||||
for _, s := range segments {
|
||||
if s == "" {
|
||||
break
|
||||
}
|
||||
path += s
|
||||
isObject, pathNotExist := er.isObject(ctx, bucket, path)
|
||||
if pathNotExist {
|
||||
return false
|
||||
}
|
||||
if er.isObject(ctx, bucket, p) {
|
||||
if isObject {
|
||||
// If there is already a file at prefix "p", return true.
|
||||
return true
|
||||
}
|
||||
// Check if there is a file as one of the parent paths.
|
||||
return isParentDirObject(path.Dir(p))
|
||||
path += slashSeparator
|
||||
}
|
||||
return isParentDirObject(parent)
|
||||
return false
|
||||
}
|
||||
|
||||
// isObject - returns `true` if the prefix is an object i.e if
|
||||
// `xl.meta` exists at the leaf, false otherwise.
|
||||
func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (ok bool) {
|
||||
func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (ok, pathDoesNotExist bool) {
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
@@ -186,5 +251,6 @@ func (er erasureObjects) isObject(ctx context.Context, bucket, prefix string) (o
|
||||
// ignored if necessary.
|
||||
readQuorum := getReadQuorum(len(storageDisks))
|
||||
|
||||
return reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum) == nil
|
||||
err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum)
|
||||
return err == nil, err == errPathNotFound
|
||||
}
|
||||
|
||||
@@ -66,6 +66,10 @@ func TestErasureParentDirIsObject(t *testing.T) {
|
||||
parentIsObject: true,
|
||||
objectName: objectName,
|
||||
},
|
||||
{
|
||||
parentIsObject: false,
|
||||
objectName: "new-dir",
|
||||
},
|
||||
{
|
||||
parentIsObject: false,
|
||||
objectName: "",
|
||||
|
||||
@@ -33,6 +33,7 @@ type parallelReader struct {
|
||||
readers []io.ReaderAt
|
||||
orgReaders []io.ReaderAt
|
||||
dataBlocks int
|
||||
errs []error
|
||||
offset int64
|
||||
shardSize int64
|
||||
shardFileSize int64
|
||||
@@ -49,6 +50,7 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
|
||||
return ¶llelReader{
|
||||
readers: readers,
|
||||
orgReaders: readers,
|
||||
errs: make([]error, len(readers)),
|
||||
dataBlocks: e.dataBlocks,
|
||||
offset: (offset / e.blockSize) * e.ShardSize(),
|
||||
shardSize: e.ShardSize(),
|
||||
@@ -169,6 +171,8 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
|
||||
// This will be communicated upstream.
|
||||
p.orgReaders[bufIdx] = nil
|
||||
p.readers[i] = nil
|
||||
p.errs[i] = err
|
||||
|
||||
// Since ReadAt returned error, trigger another read.
|
||||
readTriggerCh <- true
|
||||
return
|
||||
@@ -182,7 +186,6 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
|
||||
readerIndex++
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if p.canDecode(newBuf) {
|
||||
p.offset += p.shardSize
|
||||
if healRequired != 0 {
|
||||
@@ -191,7 +194,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
|
||||
return newBuf, nil
|
||||
}
|
||||
|
||||
return nil, errErasureReadQuorum
|
||||
return nil, reduceReadQuorumErrs(context.Background(), p.errs, objectOpIgnoredErrs, p.dataBlocks)
|
||||
}
|
||||
|
||||
type errDecodeHealRequired struct {
|
||||
|
||||
@@ -134,7 +134,7 @@ func TestErasureDecode(t *testing.T) {
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
|
||||
|
||||
bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
|
||||
bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
|
||||
}
|
||||
|
||||
writer := bytes.NewBuffer(nil)
|
||||
@@ -164,7 +164,7 @@ func TestErasureDecode(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data)
|
||||
bitrotReaders[index] = newBitrotReader(disk, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
|
||||
bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize())
|
||||
}
|
||||
for j := range disks[:test.offDisks] {
|
||||
if bitrotReaders[j] == nil {
|
||||
@@ -270,7 +270,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
|
||||
continue
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(offset, readLen, length)
|
||||
bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||
bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||
}
|
||||
err = erasure.Decode(context.Background(), buf, bitrotReaders, offset, readLen, length, nil)
|
||||
closeBitrotReaders(bitrotReaders)
|
||||
@@ -332,7 +332,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
|
||||
continue
|
||||
}
|
||||
tillOffset := erasure.ShardFileOffset(0, size, size)
|
||||
bitrotReaders[index] = newStreamingBitrotReader(disk, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||
bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
|
||||
}
|
||||
if err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil {
|
||||
panic(err)
|
||||
|
||||
@@ -99,7 +99,7 @@ func TestErasureHeal(t *testing.T) {
|
||||
readers := make([]io.ReaderAt, len(disks))
|
||||
for i, disk := range disks {
|
||||
shardFilesize := erasure.ShardFileSize(test.size)
|
||||
readers[i] = newBitrotReader(disk, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize())
|
||||
readers[i] = newBitrotReader(disk, nil, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize())
|
||||
}
|
||||
|
||||
// setup stale disks for the test case
|
||||
|
||||
@@ -186,7 +186,7 @@ func TestListOnlineDisks(t *testing.T) {
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
|
||||
partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
|
||||
fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to getLatestFileInfo %v", err)
|
||||
@@ -287,7 +287,7 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
|
||||
_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
|
||||
readQuorum := len(erasureDisks) / 2
|
||||
if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
|
||||
t.Fatalf("Failed to read xl meta data %v", reducedErr)
|
||||
@@ -295,7 +295,7 @@ func TestDisksWithAllParts(t *testing.T) {
|
||||
|
||||
// Test that all disks are returned without any failures with
|
||||
// unmodified meta data
|
||||
partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
|
||||
partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read xl meta data %v", err)
|
||||
}
|
||||
|
||||
@@ -45,12 +45,12 @@ func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts mad
|
||||
writeQuorum := getWriteQuorum(len(storageDisks))
|
||||
|
||||
// Heal bucket.
|
||||
return healBucket(ctx, storageDisks, storageEndpoints, bucket, writeQuorum, opts.DryRun)
|
||||
return healBucket(ctx, storageDisks, storageEndpoints, bucket, writeQuorum, opts)
|
||||
}
|
||||
|
||||
// Heal bucket - create buckets on disks where it does not exist.
|
||||
func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []string, bucket string, writeQuorum int,
|
||||
dryRun bool) (res madmin.HealResultItem, err error) {
|
||||
opts madmin.HealOpts) (res madmin.HealResultItem, err error) {
|
||||
|
||||
// Initialize sync waitgroup.
|
||||
g := errgroup.WithNErrs(len(storageDisks))
|
||||
@@ -84,7 +84,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
|
||||
afterState[index] = madmin.DriveStateMissing
|
||||
|
||||
// mutate only if not a dry-run
|
||||
if dryRun {
|
||||
if opts.DryRun {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -99,7 +99,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
|
||||
errs := g.Wait()
|
||||
|
||||
reducedErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, writeQuorum-1)
|
||||
if reducedErr == errVolumeNotFound {
|
||||
if errors.Is(reducedErr, errVolumeNotFound) && !opts.Recreate {
|
||||
return res, nil
|
||||
}
|
||||
|
||||
@@ -395,7 +395,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
|
||||
if latestMeta.XLV1 {
|
||||
partPath = pathJoin(object, fmt.Sprintf("part.%d", partNumber))
|
||||
}
|
||||
readers[i] = newBitrotReader(disk, bucket, partPath, tillOffset, checksumAlgo, checksumInfo.Hash, erasure.ShardSize())
|
||||
readers[i] = newBitrotReader(disk, nil, bucket, partPath, tillOffset, checksumAlgo, checksumInfo.Hash, erasure.ShardSize())
|
||||
}
|
||||
writers := make([]io.Writer, len(outDatedDisks))
|
||||
for i, disk := range outDatedDisks {
|
||||
@@ -811,7 +811,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
|
||||
storageEndpoints := er.getEndpoints()
|
||||
|
||||
// Read metadata files from all the disks
|
||||
partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID)
|
||||
partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
|
||||
|
||||
if isAllNotFound(errs) {
|
||||
err = toObjectErr(errFileNotFound, bucket, object)
|
||||
|
||||
@@ -214,7 +214,7 @@ func TestHealObjectCorrupted(t *testing.T) {
|
||||
t.Fatalf("Failed to heal object - %v", err)
|
||||
}
|
||||
|
||||
fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "")
|
||||
fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
|
||||
fi, err := getLatestFileInfo(ctx, fileInfos, errs)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to getLatestFileInfo - %v", err)
|
||||
@@ -239,7 +239,7 @@ func TestHealObjectCorrupted(t *testing.T) {
|
||||
t.Errorf("Expected nil but received %v", err)
|
||||
}
|
||||
|
||||
fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "")
|
||||
fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
|
||||
nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to getLatestFileInfo - %v", err)
|
||||
@@ -265,7 +265,7 @@ func TestHealObjectCorrupted(t *testing.T) {
|
||||
t.Errorf("Expected nil but received %v", err)
|
||||
}
|
||||
|
||||
fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "")
|
||||
fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
|
||||
nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to getLatestFileInfo - %v", err)
|
||||
|
||||
@@ -113,21 +113,9 @@ func hashOrder(key string, cardinality int) []int {
|
||||
return nums
|
||||
}
|
||||
|
||||
// Reads all `xl.meta` metadata as a FileInfo slice and checks if the data dir exists as well,
|
||||
// otherwise returns errFileNotFound (or errFileVersionNotFound)
|
||||
func getAllObjectFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string) ([]FileInfo, []error) {
|
||||
return readVersionFromDisks(ctx, disks, bucket, object, versionID, true)
|
||||
}
|
||||
|
||||
// Reads all `xl.meta` metadata as a FileInfo slice.
|
||||
// Returns error slice indicating the failed metadata reads.
|
||||
func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string) ([]FileInfo, []error) {
|
||||
return readVersionFromDisks(ctx, disks, bucket, object, versionID, false)
|
||||
}
|
||||
|
||||
// Reads all `xl.meta` metadata as a FileInfo slice and checks if the data dir
|
||||
// exists as well, if checkDataDir is set to true.
|
||||
func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, checkDataDir bool) ([]FileInfo, []error) {
|
||||
func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, readData bool) ([]FileInfo, []error) {
|
||||
metadataArray := make([]FileInfo, len(disks))
|
||||
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
@@ -138,11 +126,10 @@ func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, objec
|
||||
if disks[index] == nil {
|
||||
return errDiskNotFound
|
||||
}
|
||||
metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, checkDataDir)
|
||||
metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, readData)
|
||||
if err != nil {
|
||||
if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
|
||||
logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())
|
||||
logger.LogIf(ctx, err)
|
||||
if !IsErr(err, errFileNotFound, errVolumeNotFound, errFileVersionNotFound, errDiskNotFound) {
|
||||
logger.LogOnceIf(ctx, err, disks[index].String())
|
||||
}
|
||||
}
|
||||
return err
|
||||
|
||||
+17
-17
@@ -46,7 +46,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
|
||||
disks := er.getDisks()
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "")
|
||||
metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "", false)
|
||||
|
||||
readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
|
||||
if err != nil {
|
||||
@@ -371,7 +371,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
partsMetadata, errs = readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket,
|
||||
uploadIDPath, "")
|
||||
uploadIDPath, "", false)
|
||||
|
||||
// get Quorum for this object
|
||||
_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
|
||||
@@ -474,7 +474,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
}
|
||||
|
||||
// Read metadata again because it might be updated with parallel upload of another part.
|
||||
partsMetadata, errs = readAllFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "")
|
||||
partsMetadata, errs = readAllFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
|
||||
reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
|
||||
if reducedErr == errErasureWriteQuorum {
|
||||
return pi, toObjectErr(reducedErr, bucket, object)
|
||||
@@ -552,7 +552,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID)
|
||||
partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, opts.VersionID, false)
|
||||
|
||||
// get Quorum for this object
|
||||
readQuorum, _, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
|
||||
@@ -600,7 +600,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
|
||||
partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
|
||||
|
||||
// get Quorum for this object
|
||||
_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
|
||||
@@ -704,7 +704,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
storageDisks := er.getDisks()
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "")
|
||||
partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath, "", false)
|
||||
|
||||
// get Quorum for this object
|
||||
_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
|
||||
@@ -846,21 +846,21 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
}
|
||||
|
||||
// Check if there is any offline disk and add it to the MRF list
|
||||
for i, disk := range onlineDisks {
|
||||
if disk == nil || storageDisks[i] == nil {
|
||||
er.addPartial(bucket, object, fi.VersionID)
|
||||
break
|
||||
for _, disk := range onlineDisks {
|
||||
if disk != nil && disk.IsOnline() {
|
||||
continue
|
||||
}
|
||||
er.addPartial(bucket, object, fi.VersionID)
|
||||
break
|
||||
}
|
||||
|
||||
for i := 0; i < len(onlineDisks); i++ {
|
||||
if onlineDisks[i] == nil {
|
||||
continue
|
||||
if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
|
||||
// Object info is the same in all disks, so we can pick
|
||||
// the first meta from online disk
|
||||
fi = partsMetadata[i]
|
||||
break
|
||||
}
|
||||
// Object info is the same in all disks, so we can pick
|
||||
// the first meta from online disk
|
||||
fi = partsMetadata[i]
|
||||
break
|
||||
}
|
||||
|
||||
// Success, return object info.
|
||||
@@ -889,7 +889,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
|
||||
uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "")
|
||||
partsMetadata, errs := readAllFileInfo(ctx, er.getDisks(), minioMetaMultipartBucket, uploadIDPath, "", false)
|
||||
|
||||
// get Quorum for this object
|
||||
_, writeQuorum, err := objectQuorumFromMeta(ctx, er, partsMetadata, errs)
|
||||
|
||||
+54
-34
@@ -18,6 +18,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
@@ -58,7 +59,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
storageDisks := er.getDisks()
|
||||
metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID)
|
||||
metaArr, errs := readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, false)
|
||||
|
||||
// get Quorum for this object
|
||||
readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
|
||||
@@ -157,8 +158,16 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
|
||||
unlockOnDefer = true
|
||||
}
|
||||
|
||||
fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
|
||||
fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
|
||||
if err != nil {
|
||||
if isProxyable(ctx, bucket) && (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) {
|
||||
// proxy to replication target if active-active replication is in place.
|
||||
reader, proxy := proxyGetToReplicationTarget(ctx, bucket, object, rs, h, opts)
|
||||
if reader == nil || !proxy {
|
||||
return nil, toObjectErr(err, bucket, object)
|
||||
}
|
||||
return reader, nil
|
||||
}
|
||||
return nil, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
@@ -298,7 +307,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
|
||||
}
|
||||
checksumInfo := metaArr[index].Erasure.GetChecksumInfo(partNumber)
|
||||
partPath := pathJoin(object, metaArr[index].DataDir, fmt.Sprintf("part.%d", partNumber))
|
||||
readers[index] = newBitrotReader(disk, bucket, partPath, tillOffset,
|
||||
data := metaArr[index].Data
|
||||
readers[index] = newBitrotReader(disk, data, bucket, partPath, tillOffset,
|
||||
checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())
|
||||
|
||||
// Prefer local disks
|
||||
@@ -337,7 +347,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
|
||||
|
||||
// getObject wrapper for erasure GetObject
|
||||
func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset, length int64, writer io.Writer, opts ObjectOptions) error {
|
||||
fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
|
||||
fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
|
||||
if err != nil {
|
||||
return toObjectErr(err, bucket, object)
|
||||
}
|
||||
@@ -364,11 +374,11 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
|
||||
return er.getObjectInfo(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
|
||||
func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
|
||||
disks := er.getDisks()
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
metaArr, errs := getAllObjectFileInfo(ctx, disks, bucket, object, opts.VersionID)
|
||||
metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)
|
||||
|
||||
readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
|
||||
if err != nil {
|
||||
@@ -410,9 +420,17 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
||||
|
||||
// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
|
||||
func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
|
||||
fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts)
|
||||
fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
|
||||
if err != nil {
|
||||
// proxy HEAD to replication target if active-active replication configured on bucket
|
||||
if isProxyable(ctx, bucket) && (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) {
|
||||
oi, proxy, err := proxyHeadToReplicationTarget(ctx, bucket, object, opts)
|
||||
if proxy {
|
||||
return oi, err
|
||||
}
|
||||
}
|
||||
return objInfo, toObjectErr(err, bucket, object)
|
||||
|
||||
}
|
||||
objInfo = fi.ToObjectInfo(bucket, object)
|
||||
if objInfo.TransitionStatus == lifecycle.TransitionComplete {
|
||||
@@ -674,11 +692,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
|
||||
}
|
||||
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
if !opts.NoLock {
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
defer lk.Unlock()
|
||||
}
|
||||
defer lk.Unlock()
|
||||
|
||||
for i, w := range writers {
|
||||
if w == nil {
|
||||
@@ -727,20 +747,20 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
// Whether a disk was initially or becomes offline
|
||||
// during this upload, send it to the MRF list.
|
||||
for i := 0; i < len(onlineDisks); i++ {
|
||||
if onlineDisks[i] == nil || storageDisks[i] == nil {
|
||||
er.addPartial(bucket, object, fi.VersionID)
|
||||
break
|
||||
if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
|
||||
continue
|
||||
}
|
||||
er.addPartial(bucket, object, fi.VersionID)
|
||||
break
|
||||
}
|
||||
|
||||
for i := 0; i < len(onlineDisks); i++ {
|
||||
if onlineDisks[i] == nil {
|
||||
continue
|
||||
if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
|
||||
// Object info is the same in all disks, so we can pick
|
||||
// the first meta from online disk
|
||||
fi = partsMetadata[i]
|
||||
break
|
||||
}
|
||||
// Object info is the same in all disks, so we can pick
|
||||
// the first meta from online disk
|
||||
fi = partsMetadata[i]
|
||||
break
|
||||
}
|
||||
|
||||
return fi.ToObjectInfo(bucket, object), nil
|
||||
@@ -922,16 +942,15 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
|
||||
for _, version := range versions {
|
||||
// Check if there is any offline disk and add it to the MRF list
|
||||
for _, disk := range storageDisks {
|
||||
if disk == nil {
|
||||
// ignore delete markers for quorum
|
||||
if version.Deleted {
|
||||
continue
|
||||
}
|
||||
// all other direct versionId references we should
|
||||
// ensure no dangling file is left over.
|
||||
er.addPartial(bucket, version.Name, version.VersionID)
|
||||
break
|
||||
if disk != nil && disk.IsOnline() {
|
||||
// Skip attempted heal on online disks.
|
||||
continue
|
||||
}
|
||||
|
||||
// all other direct versionId references we should
|
||||
// ensure no dangling file is left over.
|
||||
er.addPartial(bucket, version.Name, version.VersionID)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1042,10 +1061,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
|
||||
}
|
||||
|
||||
for _, disk := range storageDisks {
|
||||
if disk == nil {
|
||||
er.addPartial(bucket, object, opts.VersionID)
|
||||
break
|
||||
if disk != nil && disk.IsOnline() {
|
||||
continue
|
||||
}
|
||||
er.addPartial(bucket, object, opts.VersionID)
|
||||
break
|
||||
}
|
||||
|
||||
return ObjectInfo{
|
||||
@@ -1071,7 +1091,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
|
||||
disks := er.getDisks()
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
|
||||
metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
|
||||
|
||||
readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
|
||||
if err != nil {
|
||||
@@ -1132,7 +1152,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
|
||||
disks := er.getDisks()
|
||||
|
||||
// Read metadata associated with the object from all disks.
|
||||
metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
|
||||
metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
|
||||
|
||||
readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
|
||||
if err != nil {
|
||||
|
||||
+114
-11
@@ -19,7 +19,9 @@ package cmd
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
crand "crypto/rand"
|
||||
"errors"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
@@ -288,16 +290,42 @@ func TestGetObjectNoQuorum(t *testing.T) {
|
||||
bucket := "bucket"
|
||||
object := "object"
|
||||
opts := ObjectOptions{}
|
||||
// Create "object" under "bucket".
|
||||
_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
|
||||
buf := make([]byte, 129<<10)
|
||||
if _, err = io.ReadFull(crand.Reader, buf); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Test use case 1: All disks are online, xl.meta are present, but data are missing
|
||||
_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Make 9 disks offline, which leaves less than quorum number of disks
|
||||
for _, disk := range xl.getDisks() {
|
||||
files, _ := disk.ListDir(ctx, bucket, object, -1)
|
||||
for _, file := range files {
|
||||
if file != "xl.meta" {
|
||||
disk.Delete(ctx, bucket, pathJoin(object, file), true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
err = xl.GetObject(ctx, bucket, object, 0, int64(len(buf)), ioutil.Discard, "", opts)
|
||||
if err != toObjectErr(errFileNotFound, bucket, object) {
|
||||
t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
|
||||
}
|
||||
|
||||
// Test use case 2: Make 9 disks offline, which leaves less than quorum number of disks
|
||||
// in a 16 disk Erasure setup. The original disks are 'replaced' with
|
||||
// naughtyDisks that fail after 'f' successful StorageAPI method
|
||||
// invocations, where f - [0,2)
|
||||
|
||||
// Create "object" under "bucket".
|
||||
_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
for f := 0; f < 2; f++ {
|
||||
diskErrors := make(map[int]error)
|
||||
for i := 0; i <= f; i++ {
|
||||
@@ -320,9 +348,84 @@ func TestGetObjectNoQuorum(t *testing.T) {
|
||||
// Fetch object from store.
|
||||
err = xl.GetObject(ctx, bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts)
|
||||
if err != toObjectErr(errErasureReadQuorum, bucket, object) {
|
||||
t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
|
||||
t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func TestHeadObjectNoQuorum(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Create an instance of xl backend.
|
||||
obj, fsDirs, err := prepareErasure16(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Cleanup backend directories.
|
||||
defer obj.Shutdown(context.Background())
|
||||
defer removeRoots(fsDirs)
|
||||
|
||||
z := obj.(*erasureServerPools)
|
||||
xl := z.serverPools[0].sets[0]
|
||||
|
||||
// Create "bucket"
|
||||
err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
bucket := "bucket"
|
||||
object := "object"
|
||||
opts := ObjectOptions{}
|
||||
|
||||
// Test use case 1: All disks are online, xl.meta are present, but data are missing
|
||||
_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, disk := range xl.getDisks() {
|
||||
files, _ := disk.ListDir(ctx, bucket, object, -1)
|
||||
for _, file := range files {
|
||||
if file != "xl.meta" {
|
||||
disk.Delete(ctx, bucket, pathJoin(object, file), true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
_, err = xl.GetObjectInfo(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
t.Errorf("Expected StatObject to succeed if data dir are not found, but failed with %v", err)
|
||||
}
|
||||
|
||||
// Test use case 2: Make 9 disks offline, which leaves less than quorum number of disks
|
||||
// in a 16 disk Erasure setup. The original disks are 'replaced' with
|
||||
// naughtyDisks that fail after 'f' successful StorageAPI method
|
||||
// invocations, where f - [0,2)
|
||||
|
||||
// Create "object" under "bucket".
|
||||
_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
erasureDisks := xl.getDisks()
|
||||
for i := range erasureDisks[:10] {
|
||||
erasureDisks[i] = nil
|
||||
}
|
||||
|
||||
z.serverPools[0].erasureDisksMu.Lock()
|
||||
xl.getDisks = func() []StorageAPI {
|
||||
return erasureDisks
|
||||
}
|
||||
z.serverPools[0].erasureDisksMu.Unlock()
|
||||
|
||||
// Fetch object from store.
|
||||
_, err = xl.GetObjectInfo(ctx, bucket, object, opts)
|
||||
if err != toObjectErr(errErasureReadQuorum, bucket, object) {
|
||||
t.Errorf("Expected getObjectInfo to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPutObjectNoQuorum(t *testing.T) {
|
||||
@@ -424,7 +527,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "")
|
||||
parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "", false)
|
||||
|
||||
parts1SC := globalStorageClass
|
||||
|
||||
@@ -437,7 +540,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "")
|
||||
parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "", false)
|
||||
parts2SC := globalStorageClass
|
||||
|
||||
// Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class
|
||||
@@ -449,7 +552,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "")
|
||||
parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "", false)
|
||||
parts3SC := globalStorageClass
|
||||
|
||||
// Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class
|
||||
@@ -467,7 +570,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "")
|
||||
parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "", false)
|
||||
parts4SC := storageclass.Config{
|
||||
Standard: storageclass.StorageClass{
|
||||
Parity: 6,
|
||||
@@ -490,7 +593,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "")
|
||||
parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "", false)
|
||||
parts5SC := storageclass.Config{
|
||||
RRS: storageclass.StorageClass{
|
||||
Parity: 2,
|
||||
@@ -512,7 +615,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "")
|
||||
parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "", false)
|
||||
parts6SC := storageclass.Config{
|
||||
RRS: storageclass.StorageClass{
|
||||
Parity: 2,
|
||||
@@ -535,7 +638,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
|
||||
t.Fatalf("Failed to putObject %v", err)
|
||||
}
|
||||
|
||||
parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "")
|
||||
parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "", false)
|
||||
parts7SC := storageclass.Config{
|
||||
Standard: storageclass.StorageClass{
|
||||
Parity: 5,
|
||||
|
||||
@@ -33,6 +33,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/cmd/config/storageclass"
|
||||
"github.com/minio/minio/cmd/logger"
|
||||
"github.com/minio/minio/pkg/color"
|
||||
"github.com/minio/minio/pkg/dsync"
|
||||
"github.com/minio/minio/pkg/madmin"
|
||||
"github.com/minio/minio/pkg/sync/errgroup"
|
||||
@@ -51,7 +52,7 @@ func (z *erasureServerPools) SingleZone() bool {
|
||||
return len(z.serverPools) == 1
|
||||
}
|
||||
|
||||
// Initialize new zone of erasure sets.
|
||||
// Initialize new pool of erasure sets.
|
||||
func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) {
|
||||
var (
|
||||
deploymentID string
|
||||
@@ -127,9 +128,9 @@ func (z *erasureServerPools) SetDriveCount() int {
|
||||
return z.serverPools[0].SetDriveCount()
|
||||
}
|
||||
|
||||
type serverPoolsAvailableSpace []zoneAvailableSpace
|
||||
type serverPoolsAvailableSpace []poolAvailableSpace
|
||||
|
||||
type zoneAvailableSpace struct {
|
||||
type poolAvailableSpace struct {
|
||||
Index int
|
||||
Available uint64
|
||||
}
|
||||
@@ -154,10 +155,10 @@ func (z *erasureServerPools) getAvailableZoneIdx(ctx context.Context, size int64
|
||||
// choose when we reach this many
|
||||
choose := rand.Uint64() % total
|
||||
atTotal := uint64(0)
|
||||
for _, zone := range serverPools {
|
||||
atTotal += zone.Available
|
||||
if atTotal > choose && zone.Available > 0 {
|
||||
return zone.Index
|
||||
for _, pool := range serverPools {
|
||||
atTotal += pool.Available
|
||||
if atTotal > choose && pool.Available > 0 {
|
||||
return pool.Index
|
||||
}
|
||||
}
|
||||
// Should not happen, but print values just in case.
|
||||
@@ -165,8 +166,8 @@ func (z *erasureServerPools) getAvailableZoneIdx(ctx context.Context, size int64
|
||||
return -1
|
||||
}
|
||||
|
||||
// getServerPoolsAvailableSpace will return the available space of each zone after storing the content.
|
||||
// If there is not enough space the zone will return 0 bytes available.
|
||||
// getServerPoolsAvailableSpace will return the available space of each pool after storing the content.
|
||||
// If there is not enough space the pool will return 0 bytes available.
|
||||
// Negative sizes are seen as 0 bytes.
|
||||
func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, size int64) serverPoolsAvailableSpace {
|
||||
if size < 0 {
|
||||
@@ -208,7 +209,7 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
|
||||
available = 0
|
||||
}
|
||||
}
|
||||
serverPools[i] = zoneAvailableSpace{
|
||||
serverPools[i] = poolAvailableSpace{
|
||||
Index: i,
|
||||
Available: available,
|
||||
}
|
||||
@@ -216,14 +217,14 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
|
||||
return serverPools
|
||||
}
|
||||
|
||||
// getZoneIdx returns the found previous object and its corresponding zone idx,
|
||||
// if none are found falls back to most available space zone.
|
||||
// getZoneIdx returns the found previous object and its corresponding pool idx,
|
||||
// if none are found falls back to most available space pool.
|
||||
func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object string, opts ObjectOptions, size int64) (idx int, err error) {
|
||||
if z.SingleZone() {
|
||||
return 0, nil
|
||||
}
|
||||
for i, zone := range z.serverPools {
|
||||
objInfo, err := zone.GetObjectInfo(ctx, bucket, object, opts)
|
||||
for i, pool := range z.serverPools {
|
||||
objInfo, err := pool.GetObjectInfo(ctx, bucket, object, opts)
|
||||
switch err.(type) {
|
||||
case ObjectNotFound:
|
||||
// VersionId was not specified but found delete marker or no versions exist.
|
||||
@@ -236,7 +237,7 @@ func (z *erasureServerPools) getZoneIdx(ctx context.Context, bucket, object stri
|
||||
}
|
||||
}
|
||||
// delete marker not specified means no versions
|
||||
// exist continue to next zone.
|
||||
// exist continue to next pool.
|
||||
if !objInfo.DeleteMarker && err != nil {
|
||||
continue
|
||||
}
|
||||
@@ -273,9 +274,24 @@ func (z *erasureServerPools) Shutdown(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) {
|
||||
func (z *erasureServerPools) BackendInfo() (b BackendInfo) {
|
||||
b.Type = BackendErasure
|
||||
|
||||
scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
|
||||
if scParity == 0 {
|
||||
scParity = z.SetDriveCount() / 2
|
||||
}
|
||||
b.StandardSCData = z.SetDriveCount() - scParity
|
||||
b.StandardSCParity = scParity
|
||||
|
||||
rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
|
||||
b.RRSCData = z.SetDriveCount() - rrSCParity
|
||||
b.RRSCParity = rrSCParity
|
||||
return
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) StorageInfo(ctx context.Context) (StorageInfo, []error) {
|
||||
var storageInfo StorageInfo
|
||||
storageInfo.Backend.Type = BackendErasure
|
||||
|
||||
storageInfos := make([]StorageInfo, len(z.serverPools))
|
||||
storageInfosErrs := make([][]error, len(z.serverPools))
|
||||
@@ -283,7 +299,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (Stora
|
||||
for index := range z.serverPools {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
storageInfos[index], storageInfosErrs[index] = z.serverPools[index].StorageInfo(ctx, local)
|
||||
storageInfos[index], storageInfosErrs[index] = z.serverPools[index].StorageInfo(ctx)
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
@@ -291,23 +307,11 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context, local bool) (Stora
|
||||
// Wait for the go routines.
|
||||
g.Wait()
|
||||
|
||||
storageInfo.Backend = z.BackendInfo()
|
||||
for _, lstorageInfo := range storageInfos {
|
||||
storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...)
|
||||
storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks)
|
||||
storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
|
||||
}
|
||||
|
||||
scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
|
||||
if scParity == 0 {
|
||||
scParity = z.SetDriveCount() / 2
|
||||
}
|
||||
|
||||
storageInfo.Backend.StandardSCData = z.SetDriveCount() - scParity
|
||||
storageInfo.Backend.StandardSCParity = scParity
|
||||
rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
|
||||
storageInfo.Backend.RRSCData = z.SetDriveCount() - rrSCParity
|
||||
storageInfo.Backend.RRSCParity = rrSCParity
|
||||
|
||||
var errs []error
|
||||
for i := range z.serverPools {
|
||||
errs = append(errs, storageInfosErrs[i]...)
|
||||
@@ -329,6 +333,12 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
|
||||
return err
|
||||
}
|
||||
|
||||
if len(allBuckets) == 0 {
|
||||
logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl")
|
||||
updates <- DataUsageInfo{} // no buckets found update data usage to reflect latest state
|
||||
return nil
|
||||
}
|
||||
|
||||
// Crawl latest allBuckets first.
|
||||
sort.Slice(allBuckets, func(i, j int) bool {
|
||||
return allBuckets[i].Created.After(allBuckets[j].Created)
|
||||
@@ -373,7 +383,7 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
|
||||
defer updateTicker.Stop()
|
||||
var lastUpdate time.Time
|
||||
|
||||
// We need to merge since we will get the same buckets from each zone.
|
||||
// We need to merge since we will get the same buckets from each pool.
|
||||
// Therefore to get the exact bucket sizes we must merge before we can convert.
|
||||
var allMerged dataUsageCache
|
||||
|
||||
@@ -474,8 +484,8 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
|
||||
|
||||
object = encodeDirObject(object)
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
gr, err = zone.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
gr, err = pool.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
@@ -497,8 +507,8 @@ func (z *erasureServerPools) GetObject(ctx context.Context, bucket, object strin
|
||||
|
||||
object = encodeDirObject(object)
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
if err := zone.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
|
||||
for _, pool := range z.serverPools {
|
||||
if err := pool.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
}
|
||||
@@ -518,8 +528,8 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
|
||||
}
|
||||
|
||||
object = encodeDirObject(object)
|
||||
for _, zone := range z.serverPools {
|
||||
objInfo, err = zone.GetObjectInfo(ctx, bucket, object, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
objInfo, err = pool.GetObjectInfo(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
@@ -535,7 +545,7 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
|
||||
return objInfo, ObjectNotFound{Bucket: bucket, Object: object}
|
||||
}
|
||||
|
||||
// PutObject - writes an object to least used erasure zone.
|
||||
// PutObject - writes an object to least used erasure pool.
|
||||
func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
// Validate put object input args.
|
||||
if err := checkPutObjectArgs(ctx, bucket, object, z); err != nil {
|
||||
@@ -553,7 +563,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
// Overwrite the object at the right zone
|
||||
// Overwrite the object at the right pool
|
||||
return z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
|
||||
}
|
||||
|
||||
@@ -567,8 +577,8 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
|
||||
if z.SingleZone() {
|
||||
return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
|
||||
}
|
||||
for _, zone := range z.serverPools {
|
||||
objInfo, err = zone.DeleteObject(ctx, bucket, object, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
objInfo, err = pool.DeleteObject(ctx, bucket, object, opts)
|
||||
if err == nil {
|
||||
return objInfo, nil
|
||||
}
|
||||
@@ -605,8 +615,8 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
|
||||
return z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
|
||||
}
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
deletedObjects, errs := zone.DeleteObjects(ctx, bucket, objects, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objects, opts)
|
||||
for i, derr := range errs {
|
||||
if derr != nil {
|
||||
derrs[i] = derr
|
||||
@@ -623,7 +633,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
|
||||
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
|
||||
|
||||
zoneIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
|
||||
poolIdx, err := z.getZoneIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
@@ -631,12 +641,12 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
if cpSrcDstSame && srcInfo.metadataOnly {
|
||||
// Version ID is set for the destination and source == destination version ID.
|
||||
if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
|
||||
return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
}
|
||||
// Destination is not versioned and source version ID is empty
|
||||
// perform an in-place update.
|
||||
if !dstOpts.Versioned && srcOpts.VersionID == "" {
|
||||
return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
}
|
||||
// Destination is versioned, source is not destination version,
|
||||
// as a special case look for if the source object is not legacy
|
||||
@@ -646,7 +656,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
// CopyObject optimization where we don't create an entire copy
|
||||
// of the content, instead we add a reference.
|
||||
srcInfo.versionOnly = true
|
||||
return z.serverPools[zoneIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -658,7 +668,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
|
||||
MTime: dstOpts.MTime,
|
||||
}
|
||||
|
||||
return z.serverPools[zoneIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
||||
return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts)
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) {
|
||||
@@ -712,14 +722,21 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
|
||||
if err != nil && err != io.EOF {
|
||||
return loi, err
|
||||
}
|
||||
loi.Objects, loi.Prefixes = merged.fileInfoVersions(bucket, prefix, delimiter, versionMarker)
|
||||
loi.IsTruncated = err == nil && len(loi.Objects) > 0
|
||||
if maxKeys > 0 && len(loi.Objects) > maxKeys {
|
||||
loi.Objects = loi.Objects[:maxKeys]
|
||||
objects := merged.fileInfoVersions(bucket, prefix, delimiter, versionMarker)
|
||||
loi.IsTruncated = err == nil && len(objects) > 0
|
||||
if maxKeys > 0 && len(objects) > maxKeys {
|
||||
objects = objects[:maxKeys]
|
||||
loi.IsTruncated = true
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.IsDir && delimiter != "" {
|
||||
loi.Prefixes = append(loi.Prefixes, obj.Name)
|
||||
} else {
|
||||
loi.Objects = append(loi.Objects, obj)
|
||||
}
|
||||
}
|
||||
if loi.IsTruncated {
|
||||
last := loi.Objects[len(loi.Objects)-1]
|
||||
last := objects[len(objects)-1]
|
||||
loi.NextMarker = encodeMarker(last.Name, merged.listID)
|
||||
loi.NextVersionIDMarker = last.VersionID
|
||||
}
|
||||
@@ -728,6 +745,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
|
||||
|
||||
func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
|
||||
var loi ListObjectsInfo
|
||||
|
||||
merged, err := z.listPath(ctx, listPathOptions{
|
||||
Bucket: bucket,
|
||||
Prefix: prefix,
|
||||
@@ -741,11 +759,24 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
|
||||
logger.LogIf(ctx, err)
|
||||
return loi, err
|
||||
}
|
||||
|
||||
// Default is recursive, if delimiter is set then list non recursive.
|
||||
loi.Objects, loi.Prefixes = merged.fileInfos(bucket, prefix, delimiter)
|
||||
loi.IsTruncated = err == nil && len(loi.Objects) > 0
|
||||
objects := merged.fileInfos(bucket, prefix, delimiter)
|
||||
loi.IsTruncated = err == nil && len(objects) > 0
|
||||
if maxKeys > 0 && len(objects) > maxKeys {
|
||||
objects = objects[:maxKeys]
|
||||
loi.IsTruncated = true
|
||||
}
|
||||
for _, obj := range objects {
|
||||
if obj.IsDir && delimiter != "" {
|
||||
loi.Prefixes = append(loi.Prefixes, obj.Name)
|
||||
} else {
|
||||
loi.Objects = append(loi.Objects, obj)
|
||||
}
|
||||
}
|
||||
if loi.IsTruncated {
|
||||
loi.NextMarker = encodeMarker(loi.Objects[len(loi.Objects)-1].Name, merged.listID)
|
||||
last := objects[len(objects)-1]
|
||||
loi.NextMarker = encodeMarker(last.Name, merged.listID)
|
||||
}
|
||||
return loi, nil
|
||||
}
|
||||
@@ -759,20 +790,20 @@ func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, p
|
||||
return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads)
|
||||
}
|
||||
|
||||
var zoneResult = ListMultipartsInfo{}
|
||||
zoneResult.MaxUploads = maxUploads
|
||||
zoneResult.KeyMarker = keyMarker
|
||||
zoneResult.Prefix = prefix
|
||||
zoneResult.Delimiter = delimiter
|
||||
for _, zone := range z.serverPools {
|
||||
result, err := zone.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
|
||||
var poolResult = ListMultipartsInfo{}
|
||||
poolResult.MaxUploads = maxUploads
|
||||
poolResult.KeyMarker = keyMarker
|
||||
poolResult.Prefix = prefix
|
||||
poolResult.Delimiter = delimiter
|
||||
for _, pool := range z.serverPools {
|
||||
result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker,
|
||||
delimiter, maxUploads)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
zoneResult.Uploads = append(zoneResult.Uploads, result.Uploads...)
|
||||
poolResult.Uploads = append(poolResult.Uploads, result.Uploads...)
|
||||
}
|
||||
return zoneResult, nil
|
||||
return poolResult, nil
|
||||
}
|
||||
|
||||
// Initiate a new multipart upload on a hashedSet based on object name.
|
||||
@@ -814,14 +845,14 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
|
||||
return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
}
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
if err == nil {
|
||||
return zone.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
return pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts)
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
// Look for information on the next zone
|
||||
// Look for information on the next pool
|
||||
continue
|
||||
}
|
||||
// Any other unhandled errors such as quorum return.
|
||||
@@ -843,14 +874,14 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
|
||||
if z.SingleZone() {
|
||||
return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
}
|
||||
for _, zone := range z.serverPools {
|
||||
mi, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
mi, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
if err == nil {
|
||||
return mi, nil
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
// upload id not found, continue to the next zone.
|
||||
// upload id not found, continue to the next pool.
|
||||
continue
|
||||
}
|
||||
// any other unhandled error return right here.
|
||||
@@ -873,10 +904,10 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
|
||||
if z.SingleZone() {
|
||||
return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
|
||||
}
|
||||
for _, zone := range z.serverPools {
|
||||
_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
if err == nil {
|
||||
return zone.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
|
||||
return pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts)
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
@@ -901,14 +932,14 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
|
||||
return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
}
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
_, err := zone.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
_, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts)
|
||||
if err == nil {
|
||||
return zone.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
return pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts)
|
||||
}
|
||||
switch err.(type) {
|
||||
case InvalidUploadID:
|
||||
// upload id not found move to next zone
|
||||
// upload id not found move to next pool
|
||||
continue
|
||||
}
|
||||
return err
|
||||
@@ -931,17 +962,17 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
|
||||
}
|
||||
|
||||
// Purge any existing object.
|
||||
for _, zone := range z.serverPools {
|
||||
zone.DeleteObject(ctx, bucket, object, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
pool.DeleteObject(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
result, err := zone.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
|
||||
for _, pool := range z.serverPools {
|
||||
result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList)
|
||||
if err != nil {
|
||||
return objInfo, err
|
||||
}
|
||||
if result.Lookup(uploadID) {
|
||||
return zone.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
return pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts)
|
||||
}
|
||||
}
|
||||
return objInfo, InvalidUploadID{
|
||||
@@ -964,8 +995,8 @@ func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string) (
|
||||
}
|
||||
return bucketInfo, nil
|
||||
}
|
||||
for _, zone := range z.serverPools {
|
||||
bucketInfo, err = zone.GetBucketInfo(ctx, bucket)
|
||||
for _, pool := range z.serverPools {
|
||||
bucketInfo, err = pool.GetBucketInfo(ctx, bucket)
|
||||
if err != nil {
|
||||
if isErrBucketNotFound(err) {
|
||||
continue
|
||||
@@ -1090,8 +1121,8 @@ func (z *erasureServerPools) ListBuckets(ctx context.Context) (buckets []BucketI
|
||||
if z.SingleZone() {
|
||||
buckets, err = z.serverPools[0].ListBuckets(ctx)
|
||||
} else {
|
||||
for _, zone := range z.serverPools {
|
||||
buckets, err = zone.ListBuckets(ctx)
|
||||
for _, pool := range z.serverPools {
|
||||
buckets, err = pool.ListBuckets(ctx)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
@@ -1125,8 +1156,8 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
|
||||
}
|
||||
|
||||
var countNoHeal int
|
||||
for _, zone := range z.serverPools {
|
||||
result, err := zone.HealFormat(ctx, dryRun)
|
||||
for _, pool := range z.serverPools {
|
||||
result, err := pool.HealFormat(ctx, dryRun)
|
||||
if err != nil && !errors.Is(err, errNoHealRequired) {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
@@ -1159,8 +1190,8 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
|
||||
// Attempt heal on the bucket metadata, ignore any failures
|
||||
_, _ = z.HealObject(ctx, minioMetaBucket, pathJoin(bucketConfigPrefix, bucket, bucketMetadataFile), "", opts)
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
result, err := zone.HealBucket(ctx, bucket, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
result, err := pool.HealBucket(ctx, bucket, opts)
|
||||
if err != nil {
|
||||
switch err.(type) {
|
||||
case BucketNotFound:
|
||||
@@ -1320,8 +1351,8 @@ func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, ver
|
||||
defer lk.RUnlock()
|
||||
}
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
result, err := zone.HealObject(ctx, bucket, object, versionID, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
@@ -1350,12 +1381,12 @@ func (z *erasureServerPools) GetMetrics(ctx context.Context) (*Metrics, error) {
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) getZoneAndSet(id string) (int, int, error) {
|
||||
for zoneIdx := range z.serverPools {
|
||||
format := z.serverPools[zoneIdx].format
|
||||
for poolIdx := range z.serverPools {
|
||||
format := z.serverPools[poolIdx].format
|
||||
for setIdx, set := range format.Erasure.Sets {
|
||||
for _, diskID := range set {
|
||||
if diskID == id {
|
||||
return zoneIdx, setIdx, nil
|
||||
return poolIdx, setIdx, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1395,12 +1426,12 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
|
||||
|
||||
for _, localDiskIDs := range diskIDs {
|
||||
for _, id := range localDiskIDs {
|
||||
zoneIdx, setIdx, err := z.getZoneAndSet(id)
|
||||
poolIdx, setIdx, err := z.getZoneAndSet(id)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
}
|
||||
erasureSetUpCount[zoneIdx][setIdx]++
|
||||
erasureSetUpCount[poolIdx][setIdx]++
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1437,16 +1468,16 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
|
||||
}
|
||||
}
|
||||
|
||||
for zoneIdx := range erasureSetUpCount {
|
||||
for setIdx := range erasureSetUpCount[zoneIdx] {
|
||||
if erasureSetUpCount[zoneIdx][setIdx] < writeQuorum {
|
||||
for poolIdx := range erasureSetUpCount {
|
||||
for setIdx := range erasureSetUpCount[poolIdx] {
|
||||
if erasureSetUpCount[poolIdx][setIdx] < writeQuorum {
|
||||
logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
|
||||
fmt.Errorf("Write quorum may be lost on zone: %d, set: %d, expected write quorum: %d",
|
||||
zoneIdx, setIdx, writeQuorum))
|
||||
fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
|
||||
poolIdx, setIdx, writeQuorum))
|
||||
return HealthResult{
|
||||
Healthy: false,
|
||||
HealingDrives: len(aggHealStateResult.HealDisks),
|
||||
ZoneID: zoneIdx,
|
||||
ZoneID: poolIdx,
|
||||
SetID: setIdx,
|
||||
WriteQuorum: writeQuorum,
|
||||
}
|
||||
@@ -1477,8 +1508,8 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
|
||||
return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts)
|
||||
}
|
||||
|
||||
for _, zone := range z.serverPools {
|
||||
err := zone.PutObjectTags(ctx, bucket, object, tags, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
err := pool.PutObjectTags(ctx, bucket, object, tags, opts)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
@@ -1506,8 +1537,8 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
|
||||
if z.SingleZone() {
|
||||
return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
|
||||
}
|
||||
for _, zone := range z.serverPools {
|
||||
err := zone.DeleteObjectTags(ctx, bucket, object, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
err := pool.DeleteObjectTags(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
@@ -1535,8 +1566,8 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
|
||||
if z.SingleZone() {
|
||||
return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
|
||||
}
|
||||
for _, zone := range z.serverPools {
|
||||
tags, err := zone.GetObjectTags(ctx, bucket, object, opts)
|
||||
for _, pool := range z.serverPools {
|
||||
tags, err := pool.GetObjectTags(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
continue
|
||||
+3
-13
@@ -481,7 +481,7 @@ func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
// ignoring errors on purpose
|
||||
storageInfos[index], _ = s.sets[index].StorageInfo(ctx, false)
|
||||
storageInfos[index], _ = s.sets[index].StorageInfo(ctx)
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
@@ -491,8 +491,6 @@ func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
|
||||
|
||||
for _, lstorageInfo := range storageInfos {
|
||||
storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...)
|
||||
storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks)
|
||||
storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
|
||||
}
|
||||
|
||||
return storageInfo
|
||||
@@ -510,7 +508,7 @@ func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
|
||||
}
|
||||
|
||||
// StorageInfo - combines output of StorageInfo across all erasure coded object sets.
|
||||
func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) {
|
||||
func (s *erasureSets) StorageInfo(ctx context.Context) (StorageInfo, []error) {
|
||||
var storageInfo StorageInfo
|
||||
|
||||
storageInfos := make([]StorageInfo, len(s.sets))
|
||||
@@ -520,7 +518,7 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
|
||||
for index := range s.sets {
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
storageInfos[index], storageInfoErrs[index] = s.sets[index].StorageInfo(ctx, local)
|
||||
storageInfos[index], storageInfoErrs[index] = s.sets[index].StorageInfo(ctx)
|
||||
return nil
|
||||
}, index)
|
||||
}
|
||||
@@ -530,14 +528,6 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
|
||||
|
||||
for _, lstorageInfo := range storageInfos {
|
||||
storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...)
|
||||
storageInfo.Backend.OnlineDisks = storageInfo.Backend.OnlineDisks.Merge(lstorageInfo.Backend.OnlineDisks)
|
||||
storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
|
||||
}
|
||||
|
||||
if local {
|
||||
// if local is true, we are not interested in the drive UUID info.
|
||||
// this is called primarily by prometheus
|
||||
return storageInfo, nil
|
||||
}
|
||||
|
||||
var errs []error
|
||||
|
||||
@@ -77,6 +77,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
|
||||
// from subsequent blocks.
|
||||
offset = 0
|
||||
}
|
||||
|
||||
// We have written all the blocks, write the last remaining block.
|
||||
if write < int64(len(block)) {
|
||||
n, err := io.Copy(dst, bytes.NewReader(block[:write]))
|
||||
@@ -89,6 +90,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
|
||||
totalWritten += n
|
||||
break
|
||||
}
|
||||
|
||||
// Copy the block.
|
||||
n, err := io.Copy(dst, bytes.NewReader(block))
|
||||
if err != nil {
|
||||
|
||||
+55
-68
@@ -109,13 +109,12 @@ func diskErrToDriveState(err error) (state string) {
|
||||
return
|
||||
}
|
||||
|
||||
// getDisksInfo - fetch disks info across all other storage API.
|
||||
func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error, onlineDisks, offlineDisks madmin.BackendDisks) {
|
||||
disksInfo = make([]madmin.Disk, len(disks))
|
||||
func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDisks madmin.BackendDisks) {
|
||||
onlineDisks = make(madmin.BackendDisks)
|
||||
offlineDisks = make(madmin.BackendDisks)
|
||||
|
||||
for _, ep := range endpoints {
|
||||
for _, disk := range disksInfo {
|
||||
ep := disk.Endpoint
|
||||
if _, ok := offlineDisks[ep]; !ok {
|
||||
offlineDisks[ep] = 0
|
||||
}
|
||||
@@ -124,6 +123,47 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for the routines.
|
||||
for _, disk := range disksInfo {
|
||||
ep := disk.Endpoint
|
||||
state := disk.State
|
||||
if state != madmin.DriveStateOk && state != madmin.DriveStateUnformatted {
|
||||
offlineDisks[ep]++
|
||||
continue
|
||||
}
|
||||
onlineDisks[ep]++
|
||||
}
|
||||
|
||||
rootDiskCount := 0
|
||||
for _, di := range disksInfo {
|
||||
if di.RootDisk {
|
||||
rootDiskCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Count offline disks as well to ensure consistent
|
||||
// reportability of offline drives on local setups.
|
||||
if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) {
|
||||
// Success.
|
||||
return onlineDisks, offlineDisks
|
||||
}
|
||||
|
||||
// Root disk should be considered offline
|
||||
for i := range disksInfo {
|
||||
ep := disksInfo[i].Endpoint
|
||||
if disksInfo[i].RootDisk {
|
||||
offlineDisks[ep]++
|
||||
onlineDisks[ep]--
|
||||
}
|
||||
}
|
||||
|
||||
return onlineDisks, offlineDisks
|
||||
}
|
||||
|
||||
// getDisksInfo - fetch disks info across all other storage API.
|
||||
func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error) {
|
||||
disksInfo = make([]madmin.Disk, len(disks))
|
||||
|
||||
g := errgroup.WithNErrs(len(disks))
|
||||
for index := range disks {
|
||||
index := index
|
||||
@@ -157,46 +197,12 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
|
||||
}, index)
|
||||
}
|
||||
|
||||
errs = g.Wait()
|
||||
// Wait for the routines.
|
||||
for i, diskInfoErr := range errs {
|
||||
ep := disksInfo[i].Endpoint
|
||||
if diskInfoErr != nil && !errors.Is(diskInfoErr, errUnformattedDisk) {
|
||||
offlineDisks[ep]++
|
||||
continue
|
||||
}
|
||||
onlineDisks[ep]++
|
||||
}
|
||||
|
||||
rootDiskCount := 0
|
||||
for _, di := range disksInfo {
|
||||
if di.RootDisk {
|
||||
rootDiskCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Count offline disks as well to ensure consistent
|
||||
// reportability of offline drives on local setups.
|
||||
if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) {
|
||||
// Success.
|
||||
return disksInfo, errs, onlineDisks, offlineDisks
|
||||
}
|
||||
|
||||
// Root disk should be considered offline
|
||||
for i := range disksInfo {
|
||||
ep := disksInfo[i].Endpoint
|
||||
if disksInfo[i].RootDisk {
|
||||
offlineDisks[ep]++
|
||||
onlineDisks[ep]--
|
||||
}
|
||||
}
|
||||
|
||||
return disksInfo, errs, onlineDisks, offlineDisks
|
||||
return disksInfo, g.Wait()
|
||||
}
|
||||
|
||||
// Get an aggregated storage info across all disks.
|
||||
func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) {
|
||||
disksInfo, errs, onlineDisks, offlineDisks := getDisksInfo(disks, endpoints)
|
||||
disksInfo, errs := getDisksInfo(disks, endpoints)
|
||||
|
||||
// Sort so that the first element is the smallest.
|
||||
sort.Sort(byDiskTotal(disksInfo))
|
||||
@@ -206,31 +212,13 @@ func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []erro
|
||||
}
|
||||
|
||||
storageInfo.Backend.Type = BackendErasure
|
||||
storageInfo.Backend.OnlineDisks = onlineDisks
|
||||
storageInfo.Backend.OfflineDisks = offlineDisks
|
||||
|
||||
return storageInfo, errs
|
||||
}
|
||||
|
||||
// StorageInfo - returns underlying storage statistics.
|
||||
func (er erasureObjects) StorageInfo(ctx context.Context, local bool) (StorageInfo, []error) {
|
||||
func (er erasureObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
|
||||
disks := er.getDisks()
|
||||
endpoints := er.getEndpoints()
|
||||
if local {
|
||||
var localDisks []StorageAPI
|
||||
var localEndpoints []string
|
||||
for i, disk := range disks {
|
||||
if disk != nil {
|
||||
if disk.IsLocal() {
|
||||
// Append this local disk since local flag is true
|
||||
localDisks = append(localDisks, disk)
|
||||
localEndpoints = append(localEndpoints, endpoints[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
disks = localDisks
|
||||
endpoints = localEndpoints
|
||||
}
|
||||
return getStorageInfo(disks, endpoints)
|
||||
}
|
||||
|
||||
@@ -242,8 +230,8 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
|
||||
return nil
|
||||
}
|
||||
|
||||
// Collect disks we can use.
|
||||
disks := er.getOnlineDisks()
|
||||
// Collect disks we can use, sorted by least inode usage.
|
||||
disks := er.getOnlineDisksSortedByUsedInodes()
|
||||
if len(disks) == 0 {
|
||||
logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl")
|
||||
return nil
|
||||
@@ -309,7 +297,6 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
|
||||
defer saverWg.Done()
|
||||
var lastSave time.Time
|
||||
|
||||
saveLoop:
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
@@ -324,17 +311,17 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
|
||||
lastSave = cache.Info.LastUpdate
|
||||
case v, ok := <-bucketResults:
|
||||
if !ok {
|
||||
break saveLoop
|
||||
// Save final state...
|
||||
cache.Info.NextCycle++
|
||||
cache.Info.LastUpdate = time.Now()
|
||||
logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
|
||||
updates <- cache
|
||||
return
|
||||
}
|
||||
cache.replace(v.Name, v.Parent, v.Entry)
|
||||
cache.Info.LastUpdate = time.Now()
|
||||
}
|
||||
}
|
||||
// Save final state...
|
||||
cache.Info.NextCycle++
|
||||
cache.Info.LastUpdate = time.Now()
|
||||
logger.LogIf(ctx, cache.save(ctx, er, dataUsageCacheName))
|
||||
updates <- cache
|
||||
}()
|
||||
|
||||
// Start one crawler per disk
|
||||
|
||||
+6
-17
@@ -271,8 +271,8 @@ func fsOpenFile(ctx context.Context, readPath string, offset int64) (io.ReadClos
|
||||
return fr, st.Size(), nil
|
||||
}
|
||||
|
||||
// Creates a file and copies data from incoming reader. Staging buffer is used by io.CopyBuffer.
|
||||
func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []byte, fallocSize int64) (int64, error) {
|
||||
// Creates a file and copies data from incoming reader.
|
||||
func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, fallocSize int64) (int64, error) {
|
||||
if filePath == "" || reader == nil {
|
||||
logger.LogIf(ctx, errInvalidArgument)
|
||||
return 0, errInvalidArgument
|
||||
@@ -317,21 +317,10 @@ func fsCreateFile(ctx context.Context, filePath string, reader io.Reader, buf []
|
||||
}
|
||||
}
|
||||
|
||||
var bytesWritten int64
|
||||
if buf != nil {
|
||||
bytesWritten, err = io.CopyBuffer(writer, reader, buf)
|
||||
if err != nil {
|
||||
if err != io.ErrUnexpectedEOF {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
} else {
|
||||
bytesWritten, err = io.Copy(writer, reader)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return 0, err
|
||||
}
|
||||
bytesWritten, err := io.Copy(writer, reader)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return bytesWritten, nil
|
||||
|
||||
@@ -75,7 +75,7 @@ func TestFSStats(t *testing.T) {
|
||||
}
|
||||
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -85,7 +85,7 @@ func TestFSStats(t *testing.T) {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "path/to/success-file"), reader, nil, 0); err != nil {
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "path/to/success-file"), reader, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -192,7 +192,7 @@ func TestFSCreateAndOpen(t *testing.T) {
|
||||
t.Fatalf("Unable to create directory, %s", err)
|
||||
}
|
||||
|
||||
if _, err = fsCreateFile(GlobalContext, "", nil, nil, 0); err != errInvalidArgument {
|
||||
if _, err = fsCreateFile(GlobalContext, "", nil, 0); err != errInvalidArgument {
|
||||
t.Fatal("Unexpected error", err)
|
||||
}
|
||||
|
||||
@@ -201,7 +201,7 @@ func TestFSCreateAndOpen(t *testing.T) {
|
||||
}
|
||||
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -229,7 +229,7 @@ func TestFSCreateAndOpen(t *testing.T) {
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
_, err = fsCreateFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), reader, nil, 0)
|
||||
_, err = fsCreateFile(GlobalContext, pathJoin(path, testCase.srcVol, testCase.srcPath), reader, 0)
|
||||
if err != testCase.expectedErr {
|
||||
t.Errorf("Test case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
|
||||
}
|
||||
@@ -258,9 +258,8 @@ func TestFSDeletes(t *testing.T) {
|
||||
t.Fatalf("Unable to create directory, %s", err)
|
||||
}
|
||||
|
||||
var buf = make([]byte, 4096)
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, buf, reader.Size()); err != nil {
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, reader.Size()); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -396,13 +395,13 @@ func TestFSRemoves(t *testing.T) {
|
||||
}
|
||||
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, nil, 0); err != nil {
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file"), reader, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
reader.Seek(0, 0)
|
||||
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file-new"), reader, nil, 0); err != nil {
|
||||
if _, err = fsCreateFile(GlobalContext, pathJoin(path, "success-vol", "success-file-new"), reader, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
// Seek back.
|
||||
@@ -515,7 +514,7 @@ func TestFSRemoveMeta(t *testing.T) {
|
||||
filePath := pathJoin(fsPath, "success-vol", "success-file")
|
||||
|
||||
var reader = bytes.NewReader([]byte("Hello, world"))
|
||||
if _, err = fsCreateFile(GlobalContext, filePath, reader, nil, 0); err != nil {
|
||||
if _, err = fsCreateFile(GlobalContext, filePath, reader, 0); err != nil {
|
||||
t.Fatalf("Unable to create file, %s", err)
|
||||
}
|
||||
|
||||
|
||||
@@ -313,14 +313,8 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
|
||||
return pi, toObjectErr(err, bucket, object)
|
||||
}
|
||||
|
||||
bufSize := int64(readSizeV1)
|
||||
if size := data.Size(); size > 0 && bufSize > size {
|
||||
bufSize = size
|
||||
}
|
||||
buf := make([]byte, bufSize)
|
||||
|
||||
tmpPartPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID+"."+mustGetUUID()+"."+strconv.Itoa(partID))
|
||||
bytesWritten, err := fsCreateFile(ctx, tmpPartPath, data, buf, data.Size())
|
||||
bytesWritten, err := fsCreateFile(ctx, tmpPartPath, data, data.Size())
|
||||
|
||||
// Delete temporary part in case of failure. If
|
||||
// PutObjectPart succeeds then there would be nothing to
|
||||
|
||||
+8
-19
@@ -200,9 +200,13 @@ func (fs *FSObjects) Shutdown(ctx context.Context) error {
|
||||
return fsRemoveAll(ctx, pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID))
|
||||
}
|
||||
|
||||
// StorageInfo - returns underlying storage statistics.
|
||||
func (fs *FSObjects) StorageInfo(ctx context.Context, _ bool) (StorageInfo, []error) {
|
||||
// BackendInfo - returns backend information
|
||||
func (fs *FSObjects) BackendInfo() BackendInfo {
|
||||
return BackendInfo{Type: BackendFS}
|
||||
}
|
||||
|
||||
// StorageInfo - returns underlying storage statistics.
|
||||
func (fs *FSObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
|
||||
atomic.AddInt64(&fs.activeIOCount, 1)
|
||||
defer func() {
|
||||
atomic.AddInt64(&fs.activeIOCount, -1)
|
||||
@@ -853,11 +857,6 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
|
||||
}
|
||||
defer reader.Close()
|
||||
|
||||
bufSize := int64(readSizeV1)
|
||||
if length > 0 && bufSize > length {
|
||||
bufSize = length
|
||||
}
|
||||
|
||||
// For negative length we read everything.
|
||||
if length < 0 {
|
||||
length = size - offset
|
||||
@@ -870,10 +869,7 @@ func (fs *FSObjects) getObject(ctx context.Context, bucket, object string, offse
|
||||
return err
|
||||
}
|
||||
|
||||
// Allocate a staging buffer.
|
||||
buf := make([]byte, int(bufSize))
|
||||
|
||||
_, err = io.CopyBuffer(writer, io.LimitReader(reader, length), buf)
|
||||
_, err = io.Copy(writer, io.LimitReader(reader, length))
|
||||
// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
|
||||
if err == io.ErrClosedPipe {
|
||||
err = nil
|
||||
@@ -1195,15 +1191,8 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
|
||||
// so that cleaning it up will be easy if the server goes down.
|
||||
tempObj := mustGetUUID()
|
||||
|
||||
// Allocate a buffer to Read() from request body
|
||||
bufSize := int64(readSizeV1)
|
||||
if size := data.Size(); size > 0 && bufSize > size {
|
||||
bufSize = size
|
||||
}
|
||||
|
||||
buf := make([]byte, int(bufSize))
|
||||
fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
|
||||
bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size())
|
||||
bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, data.Size())
|
||||
|
||||
// Delete the temporary object in the case of a
|
||||
// failure. If PutObject succeeds, then there would be
|
||||
|
||||
+3
-3
@@ -179,7 +179,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
|
||||
// Check and load TLS certificates.
|
||||
var err error
|
||||
globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
|
||||
globalPublicCerts, globalTLSCerts, globalIsTLS, err = getTLSConfig()
|
||||
logger.FatalIf(err, "Invalid TLS certificate file")
|
||||
|
||||
// Check and load Root CAs.
|
||||
@@ -211,7 +211,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
if host == "" {
|
||||
host = sortIPs(localIP4.ToSlice())[0]
|
||||
}
|
||||
return fmt.Sprintf("%s://%s", getURLScheme(globalIsSSL), net.JoinHostPort(host, globalMinioPort))
|
||||
return fmt.Sprintf("%s://%s", getURLScheme(globalIsTLS), net.JoinHostPort(host, globalMinioPort))
|
||||
}()
|
||||
|
||||
// Handle gateway specific env
|
||||
@@ -271,7 +271,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
|
||||
registerAPIRouter(router)
|
||||
|
||||
// Use all the middlewares
|
||||
router.Use(registerMiddlewares)
|
||||
router.Use(globalHandlers...)
|
||||
|
||||
var getCert certs.GetCertificateFunc
|
||||
if globalTLSCerts != nil {
|
||||
|
||||
@@ -44,7 +44,7 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
|
||||
// SSL is configured reads certification chain, prints
|
||||
// authority and expiry.
|
||||
if color.IsTerminal() && !globalCLIContext.Anonymous {
|
||||
if globalIsSSL {
|
||||
if globalIsTLS {
|
||||
printCertificateMsg(globalPublicCerts)
|
||||
}
|
||||
}
|
||||
@@ -58,10 +58,10 @@ func printGatewayCommonMsg(apiEndpoints []string) {
|
||||
apiEndpointStr := strings.Join(apiEndpoints, " ")
|
||||
|
||||
// Colorize the message and print.
|
||||
logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf(getFormatStr(len(apiEndpointStr), 1), apiEndpointStr)))
|
||||
logStartupMessage(color.Blue("Endpoint: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
|
||||
if color.IsTerminal() && !globalCLIContext.Anonymous {
|
||||
logStartupMessage(color.Blue("AccessKey: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logStartupMessage(color.Blue("SecretKey: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
|
||||
logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
|
||||
}
|
||||
printEventNotifiers()
|
||||
|
||||
|
||||
@@ -34,6 +34,11 @@ import (
|
||||
// GatewayUnsupported list of unsupported call stubs for gateway.
|
||||
type GatewayUnsupported struct{}
|
||||
|
||||
// BackendInfo returns the underlying backend information
|
||||
func (a GatewayUnsupported) BackendInfo() BackendInfo {
|
||||
return BackendInfo{Type: BackendGateway}
|
||||
}
|
||||
|
||||
// CrawlAndGetDataUsage - crawl is not implemented for gateway
|
||||
func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo) error {
|
||||
logger.CriticalIf(ctx, errors.New("not implemented"))
|
||||
|
||||
@@ -51,30 +51,19 @@ import (
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
|
||||
var (
|
||||
azureUploadChunkSize = getUploadChunkSizeFromEnv(azureChunkSizeEnvVar, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
|
||||
azureSdkTimeout = time.Duration(azureUploadChunkSize/humanize.MiByte) * azureSdkTimeoutPerMb
|
||||
azureUploadConcurrency = azureUploadMaxMemoryUsage / azureUploadChunkSize
|
||||
const (
|
||||
azureDefaultUploadChunkSizeMB = 25
|
||||
azureDownloadRetryAttempts = 5
|
||||
azureS3MinPartSize = 5 * humanize.MiByte
|
||||
metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
||||
azureMarkerPrefix = "{minio}"
|
||||
metadataPartNamePrefix = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x"
|
||||
maxPartsCount = 10000
|
||||
)
|
||||
|
||||
const (
|
||||
// The defaultDialTimeout for communicating with the cloud backends is set
|
||||
// to 30 seconds in utils.go; the Azure SDK recommends to set a timeout of 60
|
||||
// seconds per MB of data a client expects to upload so we must transfer less
|
||||
// than 0.5 MB per chunk to stay within the defaultDialTimeout tolerance.
|
||||
// See https://github.com/Azure/azure-storage-blob-go/blob/fc70003/azblob/zc_policy_retry.go#L39-L44 for more details.
|
||||
// To change the upload chunk size, set the environmental variable MINIO_AZURE_CHUNK_SIZE_MB with a (float) value between 0 and 100
|
||||
azureDefaultUploadChunkSize = 25 * humanize.MiByte
|
||||
azureSdkTimeoutPerMb = 60 * time.Second
|
||||
azureUploadMaxMemoryUsage = 100 * humanize.MiByte
|
||||
azureChunkSizeEnvVar = "MINIO_AZURE_CHUNK_SIZE_MB"
|
||||
|
||||
azureDownloadRetryAttempts = 5
|
||||
azureS3MinPartSize = 5 * humanize.MiByte
|
||||
metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
|
||||
azureMarkerPrefix = "{minio}"
|
||||
metadataPartNamePrefix = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x"
|
||||
maxPartsCount = 10000
|
||||
var (
|
||||
azureUploadChunkSize int
|
||||
azureUploadConcurrency int
|
||||
)
|
||||
|
||||
func init() {
|
||||
@@ -92,22 +81,21 @@ ENDPOINT:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for Azure Blob Storage backend on custom endpoint.
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}0.25
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}azureaccountname
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}azureaccountkey
|
||||
{{.Prompt}} {{.HelpName}} https://azureaccountname.blob.custom.azure.endpoint
|
||||
|
||||
2. Start minio gateway server for Azure Blob Storage backend with edge caching enabled.
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}azureaccountname
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}azureaccountkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}azureaccountname
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}azureaccountkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_AZURE_CHUNK_SIZE_MB {{.AssignmentOperator}}25
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
|
||||
`
|
||||
|
||||
minio.RegisterGatewayCommand(cli.Command{
|
||||
@@ -140,27 +128,6 @@ func azureGatewayMain(ctx *cli.Context) {
|
||||
minio.StartGateway(ctx, &Azure{host})
|
||||
}
|
||||
|
||||
// getUploadChunkSizeFromEnv returns the parsed chunk size from the environmental variable 'MINIO_AZURE_CHUNK_SIZE_MB'
|
||||
// The environmental variable should be a floating point number between 0 and 100 representing the MegaBytes
|
||||
// The returned value is an int representing the size in bytes
|
||||
func getUploadChunkSizeFromEnv(envvar string, defaultValue string) int {
|
||||
envChunkSize := env.Get(envvar, defaultValue)
|
||||
|
||||
i, err := strconv.ParseFloat(envChunkSize, 64)
|
||||
if err != nil {
|
||||
logger.LogIf(context.Background(), err)
|
||||
return azureDefaultUploadChunkSize
|
||||
}
|
||||
|
||||
if i <= 0 || i > 100 {
|
||||
logger.LogIf(context.Background(), fmt.Errorf("ENV '%v' should be a floating point value between 0 and 100.\n"+
|
||||
"The upload chunk size is set to its default: %s\n", azureChunkSizeEnvVar, defaultValue))
|
||||
return azureDefaultUploadChunkSize
|
||||
}
|
||||
|
||||
return int(i * humanize.MiByte)
|
||||
}
|
||||
|
||||
// Azure implements Gateway.
|
||||
type Azure struct {
|
||||
host string
|
||||
@@ -188,6 +155,20 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
|
||||
return nil, err
|
||||
}
|
||||
|
||||
azureUploadChunkSize, err = env.GetInt("MINIO_AZURE_CHUNK_SIZE_MB", azureDefaultUploadChunkSizeMB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
azureUploadChunkSize *= humanize.MiByte
|
||||
if azureUploadChunkSize <= 0 || azureUploadChunkSize > 100*humanize.MiByte {
|
||||
return nil, fmt.Errorf("MINIO_AZURE_CHUNK_SIZE_MB should be an integer value between 0 and 100")
|
||||
}
|
||||
|
||||
azureUploadConcurrency, err = env.GetInt("MINIO_AZURE_UPLOAD_CONCURRENCY", 4)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
credential, err := azblob.NewSharedKeyCredential(creds.AccessKey, creds.SecretKey)
|
||||
if err != nil {
|
||||
if _, ok := err.(base64.CorruptInputError); ok {
|
||||
@@ -208,7 +189,9 @@ func (g *Azure) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, erro
|
||||
|
||||
pipeline := azblob.NewPipeline(credential, azblob.PipelineOptions{
|
||||
Retry: azblob.RetryOptions{
|
||||
TryTimeout: azureSdkTimeout,
|
||||
// Azure SDK recommends to set a timeout of 60 seconds per MB of data so we
|
||||
// calculate here the timeout for the configured upload chunck size.
|
||||
TryTimeout: time.Duration(azureUploadChunkSize/humanize.MiByte) * 60 * time.Second,
|
||||
},
|
||||
HTTPSender: pipeline.FactoryFunc(func(next pipeline.Policy, po *pipeline.PolicyOptions) pipeline.PolicyFunc {
|
||||
return func(ctx context.Context, request pipeline.Request) (pipeline.Response, error) {
|
||||
@@ -561,7 +544,7 @@ func (a *azureObjects) Shutdown(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// StorageInfo - Not relevant to Azure backend.
|
||||
func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) {
|
||||
func (a *azureObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
|
||||
si.Backend.Type = minio.BackendGateway
|
||||
host := a.endpoint.Host
|
||||
if a.endpoint.Port() == "" {
|
||||
@@ -914,7 +897,6 @@ func (a *azureObjects) PutObject(ctx context.Context, bucket, object string, r *
|
||||
blobURL := a.client.NewContainerURL(bucket).NewBlockBlobURL(object)
|
||||
|
||||
_, err = azblob.UploadStreamToBlockBlob(ctx, data, blobURL, azblob.UploadStreamToBlockBlobOptions{
|
||||
BufferSize: azureUploadChunkSize,
|
||||
MaxBuffers: azureUploadConcurrency,
|
||||
BlobHTTPHeaders: properties,
|
||||
Metadata: metadata,
|
||||
|
||||
@@ -20,13 +20,9 @@ import (
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
minio "github.com/minio/minio/cmd"
|
||||
)
|
||||
@@ -268,36 +264,3 @@ func TestCheckAzureUploadID(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestParsingUploadChunkSize(t *testing.T) {
|
||||
key := "MINIO_AZURE_CHUNK_SIZE_MB"
|
||||
invalidValues := []string{
|
||||
"",
|
||||
"0,3",
|
||||
"100.1",
|
||||
"-1",
|
||||
}
|
||||
|
||||
for i, chunkValue := range invalidValues {
|
||||
os.Setenv(key, chunkValue)
|
||||
result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
|
||||
if result != azureDefaultUploadChunkSize {
|
||||
t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result)
|
||||
}
|
||||
}
|
||||
|
||||
validValues := []string{
|
||||
"1",
|
||||
"1.25",
|
||||
"50",
|
||||
"99",
|
||||
}
|
||||
for i, chunkValue := range validValues {
|
||||
os.Setenv(key, chunkValue)
|
||||
result := getUploadChunkSizeFromEnv(key, strconv.Itoa(azureDefaultUploadChunkSize/humanize.MiByte))
|
||||
if result == azureDefaultUploadChunkSize {
|
||||
t.Errorf("Test %d: expected: %d, got: %d", i+1, azureDefaultUploadChunkSize, result)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -112,14 +112,14 @@ GOOGLE_APPLICATION_CREDENTIALS:
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for GCS backend
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}} mygcsprojectid
|
||||
|
||||
2. Start minio gateway server for GCS backend with edge caching enabled
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} GOOGLE_APPLICATION_CREDENTIALS{{.AssignmentOperator}}/path/to/credentials.json
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*;*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
|
||||
@@ -412,7 +412,7 @@ func (l *gcsGateway) Shutdown(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// StorageInfo - Not relevant to GCS backend.
|
||||
func (l *gcsGateway) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) {
|
||||
func (l *gcsGateway) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
|
||||
si.Backend.Type = minio.BackendGateway
|
||||
si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, "storage.googleapis.com:443")
|
||||
return si, nil
|
||||
|
||||
@@ -65,13 +65,13 @@ HDFS-NAMENODE:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for HDFS backend
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}} hdfs://namenode:8200
|
||||
|
||||
2. Start minio gateway server for HDFS with edge caching enabled
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
|
||||
@@ -214,7 +214,7 @@ func (n *hdfsObjects) Shutdown(ctx context.Context) error {
|
||||
return n.clnt.Close()
|
||||
}
|
||||
|
||||
func (n *hdfsObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, errs []error) {
|
||||
func (n *hdfsObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo, errs []error) {
|
||||
fsInfo, err := n.clnt.StatFs()
|
||||
if err != nil {
|
||||
return minio.StorageInfo{}, []error{err}
|
||||
|
||||
@@ -39,13 +39,13 @@ PATH:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for NAS backend
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}} /shared/nasvol
|
||||
|
||||
2. Start minio gateway server for NAS with edge caching enabled
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
|
||||
@@ -104,8 +104,8 @@ func (n *nasObjects) IsListenSupported() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
func (n *nasObjects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) {
|
||||
si, errs := n.ObjectLayer.StorageInfo(ctx, false)
|
||||
func (n *nasObjects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
|
||||
si, errs := n.ObjectLayer.StorageInfo(ctx)
|
||||
si.Backend.GatewayOnline = si.Backend.Type == minio.BackendFS
|
||||
si.Backend.Type = minio.BackendGateway
|
||||
return si, errs
|
||||
|
||||
@@ -55,13 +55,13 @@ ENDPOINT:
|
||||
|
||||
EXAMPLES:
|
||||
1. Start minio gateway server for AWS S3 backend
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.HelpName}}
|
||||
|
||||
2. Start minio gateway server for AWS S3 backend with edge caching enabled
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ACCESS_KEY{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_SECRET_KEY{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_USER{{.AssignmentOperator}}accesskey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_ROOT_PASSWORD{{.AssignmentOperator}}secretkey
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_DRIVES{{.AssignmentOperator}}"/mnt/drive1,/mnt/drive2,/mnt/drive3,/mnt/drive4"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_EXCLUDE{{.AssignmentOperator}}"bucket1/*,*.png"
|
||||
{{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_QUOTA{{.AssignmentOperator}}90
|
||||
@@ -136,7 +136,7 @@ func randString(n int, src rand.Source, prefix string) string {
|
||||
// Chains all credential types, in the following order:
|
||||
// - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
|
||||
// - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
|
||||
// - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
|
||||
// - Static credentials provided by user (i.e. MINIO_ROOT_USER)
|
||||
var defaultProviders = []credentials.Provider{
|
||||
&credentials.EnvAWS{},
|
||||
&credentials.FileAWSCredentials{},
|
||||
@@ -274,7 +274,7 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
|
||||
}
|
||||
|
||||
// StorageInfo is not relevant to S3 backend.
|
||||
func (l *s3Objects) StorageInfo(ctx context.Context, _ bool) (si minio.StorageInfo, _ []error) {
|
||||
func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
|
||||
si.Backend.Type = minio.BackendGateway
|
||||
host := l.Client.EndpointURL().Host
|
||||
if l.Client.EndpointURL().Port() == "" {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user