mirror of
https://github.com/pgsty/minio.git
synced 2026-08-08 23:33:30 +03:00
Compare commits
438 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 04135fa6cd | |||
| 42dc6329e6 | |||
| 9b8ba97f9f | |||
| 7705605b5a | |||
| 414bcb0c73 | |||
| f4710948c4 | |||
| 3f4488c589 | |||
| 9434fff215 | |||
| 695962fae8 | |||
| 8f13c8c3bf | |||
| c1cae51fb5 | |||
| 31d16f6cc2 | |||
| a50ea92c64 | |||
| 5b2ced0119 | |||
| 8a0ba093dd | |||
| fbd8dfe60f | |||
| 60aff22931 | |||
| 5fc7da345d | |||
| fd2c38fbef | |||
| ba245c6c46 | |||
| 496027b589 | |||
| 8bd4f6568b | |||
| 9d7660b409 | |||
| da55499db0 | |||
| 22f8e39b58 | |||
| eba23bbac4 | |||
| 4550535cbb | |||
| 8432fd5ac2 | |||
| 7c948adf88 | |||
| 56b7045c20 | |||
| b1a109a611 | |||
| 7a311a3b66 | |||
| d55b6b9909 | |||
| f4389fb322 | |||
| 331208bec1 | |||
| 7680e5f81d | |||
| 6acf038a84 | |||
| bdf4e386cf | |||
| ad8a34858f | |||
| 162eced7d2 | |||
| bec1f7c26a | |||
| 8771617199 | |||
| 54bc995f0a | |||
| 6c89a81af4 | |||
| 8fa2898ff1 | |||
| 10ca0a6936 | |||
| b3314e97a6 | |||
| 3b9a948045 | |||
| 3781a0f9ad | |||
| e79b289325 | |||
| 6d4c1156d6 | |||
| 3f72c7fcc7 | |||
| d521c84d55 | |||
| 946b070744 | |||
| 4a21dce2b5 | |||
| 5fe7f9fa93 | |||
| 65f34cd823 | |||
| 196e7e072b | |||
| 6f97663174 | |||
| aed7a1818a | |||
| b50d90183e | |||
| 6b06da76cb | |||
| 6ca6788bb7 | |||
| 2e23e61a45 | |||
| 9cdf490bc5 | |||
| cfed671ea3 | |||
| 53ce92b9ca | |||
| 7350a29fec | |||
| 5cc2c62c66 | |||
| 4bc5ed6c76 | |||
| e99a597899 | |||
| 73dde66dbe | |||
| e30c0e7ca3 | |||
| 8fc200c0cc | |||
| 708296ae1b | |||
| fbb5e75e01 | |||
| f327b21557 | |||
| 45b7253f39 | |||
| 05bb655efc | |||
| 8fdfcfb562 | |||
| e7c144eeac | |||
| e98172d72d | |||
| f2d063e7b9 | |||
| a50f26b7f5 | |||
| 69294cf98a | |||
| c397fb6c7a | |||
| 961b0b524e | |||
| 860fc200b0 | |||
| 109a9e3f35 | |||
| 5f971fea6e | |||
| 0d7abe3b9f | |||
| 94fbcd8ebe | |||
| 879d5dd236 | |||
| 34187e047d | |||
| 0ee722f8c3 | |||
| b7d11141e1 | |||
| e9babf3dac | |||
| 0bb81f2e9c | |||
| bea0b050cd | |||
| ce62980d4e | |||
| dc88865908 | |||
| 9fbd931058 | |||
| b0264bdb90 | |||
| 95d6f43cc8 | |||
| 9cb94eb4a9 | |||
| bd0819330d | |||
| 8d9e83fd99 | |||
| be02333529 | |||
| 506f121576 | |||
| ca488cce87 | |||
| 11dc723324 | |||
| dd6ea18901 | |||
| 3369eeb920 | |||
| 9032f49f25 | |||
| fbc6f3f6e8 | |||
| fba883839d | |||
| a93214ea63 | |||
| e6b0fc465b | |||
| 70fbcfee4a | |||
| 0b074d0fae | |||
| d67e4d5b17 | |||
| 891c60d83d | |||
| fe3e49c4eb | |||
| 58306a9d34 | |||
| 41091d9472 | |||
| a4cfb5e1ed | |||
| 51aa59a737 | |||
| 8bedb419a9 | |||
| f56a182b71 | |||
| 317b40ef90 | |||
| e938ece492 | |||
| 02331a612c | |||
| 8317557f70 | |||
| 1bb7a2a295 | |||
| 215ca58d6a | |||
| 12f570a307 | |||
| e4b619ce1a | |||
| 0a286153bb | |||
| 22d59e757d | |||
| 0daa2dbf59 | |||
| 96c2304ae8 | |||
| 343dd2f491 | |||
| 38f35463b7 | |||
| 5573986e8e | |||
| f3367a1b20 | |||
| a3c2f7b0e8 | |||
| 8fbec30998 | |||
| a7466eeb0e | |||
| 8b1e819bf3 | |||
| fe63664164 | |||
| 4598827dcb | |||
| 9afdb05bf4 | |||
| 9569a85cee | |||
| 54721b7c7b | |||
| 91d8bddbd1 | |||
| 80adc87a14 | |||
| 117ad1b65b | |||
| 2229509362 | |||
| 6ef8e87492 | |||
| 0a25083fdb | |||
| 15137d0327 | |||
| 8c9974bc0f | |||
| 079b6c2b50 | |||
| 0924b34a17 | |||
| 754f7a8a39 | |||
| 64bafe1dfe | |||
| c3e456e7e6 | |||
| 57aaeafd2f | |||
| 3c2e1a87e2 | |||
| da95a2d13f | |||
| cc5e05fdeb | |||
| a79c390cca | |||
| 8a56af439c | |||
| f6e581ce54 | |||
| 8953f88780 | |||
| 4b4a98d5e5 | |||
| 7472818d94 | |||
| ad44fe8d3e | |||
| 55e713db0a | |||
| 33322e6638 | |||
| a1792ca0d1 | |||
| ac8c43fe9c | |||
| 4d40ee00e9 | |||
| 06f59ad631 | |||
| 877e0cac03 | |||
| ef67c39910 | |||
| 508710f4d1 | |||
| 3aa3d9cf14 | |||
| c2fedb4c3f | |||
| 03dc65e12d | |||
| b8d62a8068 | |||
| dbc2368a7b | |||
| 54aed421b8 | |||
| d5e8dac1cf | |||
| 96ec8fcba1 | |||
| 0663eb69ed | |||
| 0594d37230 | |||
| 3cc30bcc18 | |||
| 99c1a642a4 | |||
| c60f54e5be | |||
| 483389f2e2 | |||
| c0f2f84285 | |||
| 069d118329 | |||
| a7b1834772 | |||
| 6415dec37a | |||
| 74253e1ddc | |||
| 01b3fb91e5 | |||
| 2dc917e87f | |||
| 0a284a1a10 | |||
| 5c8339e1e8 | |||
| fd37418da2 | |||
| bbfea29c2b | |||
| aa703dc903 | |||
| 8cd80fec8c | |||
| 780882efcf | |||
| c5636143c6 | |||
| ba6218b354 | |||
| 8e32de3ba9 | |||
| e37508fb8f | |||
| b46a717425 | |||
| 7926df0b80 | |||
| 557df666fd | |||
| f91b257f50 | |||
| 28a2d1eb3d | |||
| a0ae1489e5 | |||
| edfb310a59 | |||
| a2312028b9 | |||
| 78f1f69d57 | |||
| e1e33077e8 | |||
| b3e7de010d | |||
| f5b04865f4 | |||
| bf1c6edb76 | |||
| 2ac7fee017 | |||
| 128256e3ab | |||
| a66a7f3e97 | |||
| 20b79f8945 | |||
| 9a877734b2 | |||
| 409c391850 | |||
| 763ff085a6 | |||
| 5b9656374c | |||
| b32014549c | |||
| 9476d212bc | |||
| 000928d34e | |||
| 6829ae5b13 | |||
| f09756443d | |||
| 5512016885 | |||
| 21ecb941fe | |||
| 77e94087cf | |||
| 9ab1f25a47 | |||
| aaab7aefbe | |||
| 5b8599e52d | |||
| 74e0c9ab9b | |||
| dcce83b288 | |||
| f731e7ea36 | |||
| ec30bb89a4 | |||
| 7cd08594f6 | |||
| 2b4531f069 | |||
| 11544a62aa | |||
| 18550387d5 | |||
| efb03e19e6 | |||
| c27d0583d4 | |||
| 0de2b9a1b2 | |||
| 9dc29d7687 | |||
| 72871dbb9a | |||
| 4bda4e4e2b | |||
| 1971c54a50 | |||
| bb77b89da0 | |||
| b336e9a79f | |||
| a2ab21e91c | |||
| 603437e70f | |||
| db3a9a5990 | |||
| 24c7e73b4e | |||
| c053e57068 | |||
| 6d20ec3bea | |||
| c50627ee3e | |||
| b3cd893f93 | |||
| 22d2dbc4e6 | |||
| 2b5d9428b1 | |||
| d6446cb096 | |||
| c34bdc33fb | |||
| dd8547e51c | |||
| aec023f537 | |||
| e101eeeda9 | |||
| f29522269d | |||
| 3c470a6b8b | |||
| 6bc7d711b3 | |||
| 10d5dd3a67 | |||
| d9f1df01eb | |||
| cdeab19673 | |||
| 22ee678136 | |||
| 50a8f13e85 | |||
| 6dec60b6e6 | |||
| ac3a19138a | |||
| 21e8e071d7 | |||
| 57f84a8b4c | |||
| 8a672e70a7 | |||
| 22041bbcc4 | |||
| 5afb459113 | |||
| 91ebac0a00 | |||
| 5fcb1cfd31 | |||
| 3a90fb108c | |||
| 4eeb48f8e0 | |||
| 373d48c8a3 | |||
| 1472875670 | |||
| 74cfb207c1 | |||
| 6a096e7dc7 | |||
| 9788d85ea3 | |||
| 69c0e18685 | |||
| 3cac927348 | |||
| 9081346c40 | |||
| fcfadb0e51 | |||
| 2add57cfed | |||
| c5279ec630 | |||
| b73699fad8 | |||
| b8ebe54e53 | |||
| c3d70e0795 | |||
| 8c4561b8da | |||
| fd421ddd6f | |||
| 9947c01c8e | |||
| a00db4267c | |||
| 36385010f5 | |||
| fa6d082bfd | |||
| 9fab91852a | |||
| b733e6e83c | |||
| ce05bb69dc | |||
| 37aa5934a1 | |||
| 1647fc7edc | |||
| 7b92687397 | |||
| 419e5baf16 | |||
| dc48cd841a | |||
| b0e1776d6d | |||
| 7a7068ee47 | |||
| cbc0ef459b | |||
| a2aabfabd9 | |||
| 822cbd4b43 | |||
| 3c19a9308d | |||
| 32890342ce | |||
| ed2c2a285f | |||
| 18e23bafd9 | |||
| 8b8be2695f | |||
| 96fbf18201 | |||
| c8a57a8fa2 | |||
| b1c2dacab3 | |||
| 65939913b4 | |||
| 08b3a466e8 | |||
| 5aa7c38035 | |||
| 1df5e31706 | |||
| 9f7044aed0 | |||
| 41de53996b | |||
| 9878031cfd | |||
| e3fbcaeb72 | |||
| ca6dd8be5e | |||
| fba0924b1d | |||
| 703ed46d79 | |||
| f7ca6c63c2 | |||
| ad69b9907f | |||
| b9269151a4 | |||
| bfddbb8b40 | |||
| 13a2dc8485 | |||
| 1e51424e8a | |||
| 5b114b43f7 | |||
| 812f5a02d7 | |||
| 19f70dbfbf | |||
| 1c99fb106c | |||
| 71c32e9b48 | |||
| 380a59520b | |||
| 3995355150 | |||
| 8208bcb896 | |||
| d665e855de | |||
| 18b3655c99 | |||
| 6a8d8f34a5 | |||
| b1c1f02132 | |||
| ea93643e6a | |||
| e47e625f73 | |||
| b13fcaf666 | |||
| 9458485e43 | |||
| 0ce9e00ffa | |||
| c778c381b5 | |||
| 0d1fbef751 | |||
| b48bbe08b2 | |||
| cce90cb2b7 | |||
| 07b1281046 | |||
| 3515b99671 | |||
| 6a67c277eb | |||
| 1067dd3011 | |||
| 7cafdc0512 | |||
| 8a57b6bced | |||
| 6f0ed2a091 | |||
| 53abd25116 | |||
| 1ea7826c0e | |||
| 97f4cf48f8 | |||
| 0cde37be50 | |||
| 6aeca54ece | |||
| 124e28578c | |||
| 62c9e500de | |||
| 02cc18ff29 | |||
| ba4566e86d | |||
| 87cb0081ec | |||
| 4a6af93c83 | |||
| a2f0771fd3 | |||
| af564b8ba0 | |||
| adb8be069e | |||
| 7c8746732b | |||
| f506117edb | |||
| 1c5af7c31a | |||
| 3a0125fa1f | |||
| 328cb0a076 | |||
| c3c8441a1d | |||
| fa2a8d7209 | |||
| e3ea97c964 | |||
| 7219ae530e | |||
| 8f8f8854f0 | |||
| 4c6869cd9a | |||
| bc7c0d8624 | |||
| 11dfc817f3 | |||
| dde1a12819 | |||
| 065fd094d1 | |||
| d09351bb10 | |||
| 25d38e030b | |||
| 9ebd10d3f4 | |||
| 8a9b886011 | |||
| 21f0d6b549 | |||
| 3ba927edae | |||
| c4ca0a5a57 | |||
| 406ea4f281 | |||
| 64aa7feabd | |||
| 875f4076ec | |||
| 4643efe6be | |||
| b760137e1d | |||
| 5f56f441bf | |||
| 96a22bfcbb | |||
| 6c59b33fb1 | |||
| dfaf735073 | |||
| 0d2b7bf94d | |||
| 7fcfde7f07 | |||
| b1391d1991 | |||
| 49c8e16410 | |||
| 0e93681589 |
+9
-2
@@ -1,10 +1,17 @@
|
||||
.git
|
||||
.github
|
||||
docs
|
||||
default.etcd
|
||||
*.gz
|
||||
*.tar.gz
|
||||
*.bzip2
|
||||
*.zip
|
||||
browser/node_modules
|
||||
node_modules
|
||||
node_modules
|
||||
docs/debugging/s3-verify/s3-verify
|
||||
docs/debugging/xl-meta/xl-meta
|
||||
docs/debugging/s3-check-md5/s3-check-md5
|
||||
docs/debugging/hash-set/hash-set
|
||||
docs/debugging/healing-bin/healing-bin
|
||||
docs/debugging/inspect/inspect
|
||||
docs/debugging/pprofgoparser/pprofgoparser
|
||||
docs/debugging/reorder-disks/reorder-disks
|
||||
|
||||
@@ -1,3 +1,9 @@
|
||||
## Community Contribution License
|
||||
All community contributions in this pull request are licensed to the project maintainers
|
||||
under the terms of the [Apache 2 license](https://www.apache.org/licenses/LICENSE-2.0).
|
||||
By creating this pull request I represent that I have the right to license the
|
||||
contributions to the project maintainers under the Apache 2 license.
|
||||
|
||||
## Description
|
||||
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
os: [ubuntu-latest, windows-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
os: [ubuntu-latest]
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -61,7 +62,7 @@ jobs:
|
||||
# are turned off - i.e. if ldap="", then ldap server is not enabled for
|
||||
# the tests.
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
ldap: ["", "localhost:389"]
|
||||
etcd: ["", "http://localhost:2379"]
|
||||
openid: ["", "http://127.0.0.1:5556/dex"]
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -29,7 +30,7 @@ jobs:
|
||||
- name: setup-go-step
|
||||
uses: actions/setup-go@v2
|
||||
with:
|
||||
go-version: 1.20.x
|
||||
go-version: 1.21.x
|
||||
|
||||
- name: github sha short
|
||||
id: vars
|
||||
@@ -37,7 +38,11 @@ jobs:
|
||||
|
||||
- name: build-minio
|
||||
run: |
|
||||
TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
|
||||
TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
|
||||
|
||||
- name: multipart uploads test
|
||||
run: |
|
||||
${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
|
||||
|
||||
- name: compress and encrypt
|
||||
run: |
|
||||
@@ -59,7 +64,14 @@ jobs:
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
|
||||
done
|
||||
docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
|
||||
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
|
||||
docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
|
||||
for volume in $(docker volume ls -q | grep minio); do
|
||||
docker volume rm ${volume} || true
|
||||
done
|
||||
|
||||
docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: minio/minio:${JOB_NAME}
|
||||
image: quay.io/minio/minio:${JOB_NAME}
|
||||
command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
|
||||
expose:
|
||||
- "9000"
|
||||
@@ -16,10 +16,10 @@ x-minio-common: &minio-common
|
||||
MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: minio/minio:${JOB_NAME}
|
||||
image: quay.io/minio/minio:${JOB_NAME}
|
||||
command: server --console-address ":9001" edata{1...4}
|
||||
expose:
|
||||
- "9000"
|
||||
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
@@ -2,7 +2,7 @@ version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: minio/minio:${JOB_NAME}
|
||||
image: quay.io/minio/minio:${JOB_NAME}
|
||||
command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
|
||||
expose:
|
||||
- "9000"
|
||||
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${RELEASE}
|
||||
command: server http://site1-minio{1...4}/data{1...2}
|
||||
environment:
|
||||
- MINIO_PROMETHEUS_AUTH_TYPE=public
|
||||
- CI=true
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
site1-minio1:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio1
|
||||
volumes:
|
||||
- site1-data1-1:/data1
|
||||
- site1-data1-2:/data2
|
||||
|
||||
site1-minio2:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio2
|
||||
volumes:
|
||||
- site1-data2-1:/data1
|
||||
- site1-data2-2:/data2
|
||||
|
||||
site1-minio3:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio3
|
||||
volumes:
|
||||
- site1-data3-1:/data1
|
||||
- site1-data3-2:/data2
|
||||
|
||||
site1-minio4:
|
||||
<<: *minio-common
|
||||
hostname: site1-minio4
|
||||
volumes:
|
||||
- site1-data4-1:/data1
|
||||
- site1-data4-2:/data2
|
||||
|
||||
site1-nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: site1-nginx
|
||||
volumes:
|
||||
- ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9001:9001"
|
||||
depends_on:
|
||||
- site1-minio1
|
||||
- site1-minio2
|
||||
- site1-minio3
|
||||
- site1-minio4
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
site1-data1-1:
|
||||
site1-data1-2:
|
||||
site1-data2-1:
|
||||
site1-data2-2:
|
||||
site1-data3-1:
|
||||
site1-data3-2:
|
||||
site1-data4-1:
|
||||
site1-data4-2:
|
||||
@@ -0,0 +1,66 @@
|
||||
version: '3.7'
|
||||
|
||||
# Settings and configurations that are common for all containers
|
||||
x-minio-common: &minio-common
|
||||
image: quay.io/minio/minio:${RELEASE}
|
||||
command: server http://site2-minio{1...4}/data{1...2}
|
||||
environment:
|
||||
- MINIO_PROMETHEUS_AUTH_TYPE=public
|
||||
- CI=true
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
# it through port 9000.
|
||||
services:
|
||||
site2-minio1:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio1
|
||||
volumes:
|
||||
- site2-data1-1:/data1
|
||||
- site2-data1-2:/data2
|
||||
|
||||
site2-minio2:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio2
|
||||
volumes:
|
||||
- site2-data2-1:/data1
|
||||
- site2-data2-2:/data2
|
||||
|
||||
site2-minio3:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio3
|
||||
volumes:
|
||||
- site2-data3-1:/data1
|
||||
- site2-data3-2:/data2
|
||||
|
||||
site2-minio4:
|
||||
<<: *minio-common
|
||||
hostname: site2-minio4
|
||||
volumes:
|
||||
- site2-data4-1:/data1
|
||||
- site2-data4-2:/data2
|
||||
|
||||
site2-nginx:
|
||||
image: nginx:1.19.2-alpine
|
||||
hostname: site2-nginx
|
||||
volumes:
|
||||
- ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
|
||||
ports:
|
||||
- "9002:9002"
|
||||
depends_on:
|
||||
- site2-minio1
|
||||
- site2-minio2
|
||||
- site2-minio3
|
||||
- site2-minio4
|
||||
|
||||
## By default this config uses default local driver,
|
||||
## For custom volumes replace with volume driver configuration.
|
||||
volumes:
|
||||
site2-data1-1:
|
||||
site2-data1-2:
|
||||
site2-data2-1:
|
||||
site2-data2-2:
|
||||
site2-data3-1:
|
||||
site2-data3-2:
|
||||
site2-data4-1:
|
||||
site2-data4-2:
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
|
||||
## change working directory
|
||||
cd .github/workflows/multipart/
|
||||
|
||||
function cleanup() {
|
||||
docker-compose -f docker-compose-site1.yaml rm -s -f || true
|
||||
docker-compose -f docker-compose-site2.yaml rm -s -f || true
|
||||
for volume in $(docker volume ls -q | grep minio); do
|
||||
docker volume rm ${volume} || true
|
||||
done
|
||||
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
docker volume rm $(docker volume ls -q -f dangling=true) || true
|
||||
}
|
||||
|
||||
cleanup
|
||||
|
||||
if [ ! -f ./mc ]; then
|
||||
wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
|
||||
chmod +x mc
|
||||
fi
|
||||
|
||||
(
|
||||
cd /tmp
|
||||
go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
|
||||
)
|
||||
|
||||
export RELEASE=RELEASE.2023-08-29T23-07-35Z
|
||||
|
||||
docker-compose -f docker-compose-site1.yaml up -d
|
||||
docker-compose -f docker-compose-site2.yaml up -d
|
||||
|
||||
sleep 30s
|
||||
|
||||
./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
|
||||
./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
|
||||
|
||||
./mc ready site1/
|
||||
./mc ready site2/
|
||||
|
||||
./mc admin replicate add site1 site2
|
||||
./mc mb site1/testbucket/
|
||||
./mc cp -r --quiet /usr/bin site1/testbucket/
|
||||
|
||||
sleep 5
|
||||
|
||||
s3-check-md5 -h
|
||||
|
||||
failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
|
||||
if [ $failed_count_site1 -ne 0 ]; then
|
||||
echo "failed with multipart on site1 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $failed_count_site2 -ne 0 ]; then
|
||||
echo "failed with multipart on site2 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
./mc cp -r --quiet /usr/bin site1/testbucket/
|
||||
|
||||
sleep 5
|
||||
|
||||
failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
|
||||
## we do not need to fail here, since we are going to test
|
||||
## upgrading to master, healing and being able to recover
|
||||
## the last version.
|
||||
if [ $failed_count_site1 -ne 0 ]; then
|
||||
echo "failed with multipart on site1 uploads ${failed_count_site1}"
|
||||
fi
|
||||
|
||||
if [ $failed_count_site2 -ne 0 ]; then
|
||||
echo "failed with multipart on site2 uploads ${failed_count_site2}"
|
||||
fi
|
||||
|
||||
export RELEASE=${1}
|
||||
|
||||
docker-compose -f docker-compose-site1.yaml up -d
|
||||
docker-compose -f docker-compose-site2.yaml up -d
|
||||
|
||||
./mc ready site1/
|
||||
./mc ready site2/
|
||||
|
||||
for i in $(seq 1 10); do
|
||||
# mc admin heal -r --remove when used against a LB endpoint
|
||||
# behaves flaky, let this run 10 times before giving up
|
||||
./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
|
||||
./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
|
||||
done
|
||||
|
||||
failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
|
||||
|
||||
if [ $failed_count_site1 -ne 0 ]; then
|
||||
echo "failed with multipart on site1 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $failed_count_site2 -ne 0 ]; then
|
||||
echo "failed with multipart on site2 uploads"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cleanup
|
||||
|
||||
## change working directory
|
||||
cd ../../../
|
||||
@@ -0,0 +1,61 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server site1-minio1:9000;
|
||||
server site1-minio2:9000;
|
||||
server site1-minio3:9000;
|
||||
server site1-minio4:9000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9001;
|
||||
listen [::]:9001;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
user nginx;
|
||||
worker_processes auto;
|
||||
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
pid /var/run/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 4096;
|
||||
}
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
sendfile on;
|
||||
keepalive_timeout 65;
|
||||
|
||||
# include /etc/nginx/conf.d/*.conf;
|
||||
|
||||
upstream minio {
|
||||
server site2-minio1:9000;
|
||||
server site2-minio2:9000;
|
||||
server site2-minio3:9000;
|
||||
server site2-minio4:9000;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 9002;
|
||||
listen [::]:9002;
|
||||
server_name localhost;
|
||||
|
||||
# To allow special characters in headers
|
||||
ignore_invalid_headers off;
|
||||
# Allow any size file to be uploaded.
|
||||
# Set to a value such as 1000m; to restrict file size to a specific value
|
||||
client_max_body_size 0;
|
||||
# To disable buffering
|
||||
proxy_buffering off;
|
||||
proxy_request_buffering off;
|
||||
|
||||
location / {
|
||||
proxy_set_header Host $http_host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
proxy_connect_timeout 300;
|
||||
# Default is HTTP/1, keepalive is only enabled in HTTP/1.1
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection "";
|
||||
chunked_transfer_encoding off;
|
||||
|
||||
proxy_pass http://minio;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -21,7 +22,7 @@ jobs:
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
@@ -35,6 +36,12 @@ jobs:
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-decom
|
||||
|
||||
- name: Test Config File
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-configfile
|
||||
|
||||
- name: Test Replication
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
@@ -47,3 +54,8 @@ jobs:
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-site-replication-minio
|
||||
|
||||
- name: Test Versioning
|
||||
run: |
|
||||
sudo sysctl net.ipv6.conf.all.disable_ipv6=0
|
||||
sudo sysctl net.ipv6.conf.default.disable_ipv6=0
|
||||
make test-versioning
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -16,7 +16,7 @@ docker volume rm $(docker volume ls -f dangling=true) || true
|
||||
cd .github/workflows/mint
|
||||
|
||||
docker-compose -f minio-${MODE}.yaml up -d
|
||||
sleep 5m
|
||||
sleep 30s
|
||||
|
||||
docker system prune -f || true
|
||||
docker volume prune -f || true
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
runs-on: ${{ matrix.os }}
|
||||
strategy:
|
||||
matrix:
|
||||
go-version: [1.20.x]
|
||||
go-version: [1.21.x]
|
||||
os: [ubuntu-latest]
|
||||
|
||||
steps:
|
||||
|
||||
@@ -3,6 +3,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
@@ -20,7 +21,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.19.12
|
||||
go-version: 1.21.5
|
||||
check-latest: true
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
||||
+5
-2
@@ -32,6 +32,8 @@ minio.RELEASE*
|
||||
mc
|
||||
nancy
|
||||
inspects/*
|
||||
.bin/
|
||||
*.gz
|
||||
docs/debugging/s3-verify/s3-verify
|
||||
docs/debugging/xl-meta/xl-meta
|
||||
docs/debugging/s3-check-md5/s3-check-md5
|
||||
@@ -39,5 +41,6 @@ docs/debugging/hash-set/hash-set
|
||||
docs/debugging/healing-bin/healing-bin
|
||||
docs/debugging/inspect/inspect
|
||||
docs/debugging/pprofgoparser/pprofgoparser
|
||||
.bin/
|
||||
*.gz
|
||||
docs/debugging/reorder-disks/reorder-disks
|
||||
docs/debugging/populate-hard-links/populate-hardlinks
|
||||
docs/debugging/xattr/xattr
|
||||
+3
-1
@@ -13,7 +13,6 @@ linters:
|
||||
enable:
|
||||
- durationcheck
|
||||
- gocritic
|
||||
- gofmt
|
||||
- gofumpt
|
||||
- goimports
|
||||
- gomodguard
|
||||
@@ -30,5 +29,8 @@ linters:
|
||||
issues:
|
||||
exclude-use-default: false
|
||||
exclude:
|
||||
- "empty-block:"
|
||||
- "unused-parameter:"
|
||||
- "dot-imports:"
|
||||
- should have a package comment
|
||||
- error strings should not be capitalized or end with punctuation or a newline
|
||||
|
||||
+1
-3
@@ -1,8 +1,6 @@
|
||||
FROM minio/minio:latest
|
||||
|
||||
ENV PATH=/opt/bin:$PATH
|
||||
|
||||
COPY ./minio /opt/bin/minio
|
||||
COPY ./minio /usr/bin/minio
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
+35
-23
@@ -1,4 +1,31 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
@@ -17,33 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
PATH=/opt/bin:$PATH
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||
|
||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
|
||||
RUN \
|
||||
microdnf clean all && \
|
||||
microdnf update --nodocs && \
|
||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
|
||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
|
||||
microdnf clean all && \
|
||||
chmod +x /opt/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/verify-minio.sh && \
|
||||
/usr/bin/verify-minio.sh && \
|
||||
microdnf clean all
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
|
||||
+33
-25
@@ -1,6 +1,31 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
@@ -19,35 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
PATH=/opt/bin:$PATH
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||
|
||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
|
||||
RUN \
|
||||
microdnf clean all && \
|
||||
microdnf update --nodocs && \
|
||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
|
||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
|
||||
microdnf clean all && \
|
||||
chmod +x /opt/bin/minio && \
|
||||
chmod +x /opt/bin/mc && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/verify-minio.sh && \
|
||||
/usr/bin/verify-minio.sh && \
|
||||
microdnf clean all
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
|
||||
+26
-24
@@ -1,6 +1,25 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
@@ -18,34 +37,17 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
PATH=/opt/bin:$PATH
|
||||
MINIO_CONFIG_ENV_FILE=config.env
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
|
||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
|
||||
RUN \
|
||||
microdnf clean all && \
|
||||
microdnf update --nodocs && \
|
||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
|
||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
|
||||
microdnf clean all && \
|
||||
chmod +x /opt/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/verify-minio.sh && \
|
||||
/usr/bin/verify-minio.sh && \
|
||||
microdnf clean all
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.1
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi8/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
LABEL name="MinIO" \
|
||||
vendor="MinIO Inc <dev@min.io>" \
|
||||
maintainer="MinIO Inc <dev@min.io>" \
|
||||
version="${RELEASE}" \
|
||||
release="${RELEASE}" \
|
||||
summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
|
||||
description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
|
||||
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
@@ -6,9 +6,8 @@ GOARCH := $(shell go env GOARCH)
|
||||
GOOS := $(shell go env GOOS)
|
||||
|
||||
VERSION ?= $(shell git describe --tags)
|
||||
TAG ?= "minio/minio:$(VERSION)"
|
||||
TAG ?= "quay.io/minio/minio:$(VERSION)"
|
||||
|
||||
GOLANGCI_VERSION = v1.51.2
|
||||
GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
|
||||
GOLANGCI = $(GOLANGCI_DIR)/golangci-lint
|
||||
|
||||
@@ -23,8 +22,8 @@ help: ## print this help
|
||||
|
||||
getdeps: ## fetch necessary dependencies
|
||||
@mkdir -p ${GOPATH}/bin
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) $(GOLANGCI_VERSION)
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
|
||||
@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR)
|
||||
@echo "Installing msgp" && go install -v github.com/tinylib/msgp@6ac204f0b4d48d17ab4fa442134c7fba13127a4e
|
||||
@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
|
||||
|
||||
crosscompile: ## cross compile minio
|
||||
@@ -45,22 +44,29 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
|
||||
@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
|
||||
|
||||
check: test
|
||||
test: verifiers build ## builds minio, runs linters, tests
|
||||
test: verifiers build build-debugging ## builds minio, runs linters, tests
|
||||
@echo "Running unit tests"
|
||||
@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
|
||||
|
||||
test-root-disable: install
|
||||
test-root-disable: install-race
|
||||
@echo "Running minio root lockdown tests"
|
||||
@env bash $(PWD)/buildscripts/disable-root.sh
|
||||
|
||||
test-decom: install
|
||||
test-decom: install-race
|
||||
@echo "Running minio decom tests"
|
||||
@env bash $(PWD)/docs/distributed/decom.sh
|
||||
@env bash $(PWD)/docs/distributed/decom-encrypted.sh
|
||||
@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
|
||||
@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
|
||||
|
||||
test-upgrade: build
|
||||
test-versioning: install-race
|
||||
@echo "Running minio versioning tests"
|
||||
@env bash $(PWD)/docs/bucket/versioning/versioning-tests.sh
|
||||
|
||||
test-configfile: install-race
|
||||
@env bash $(PWD)/docs/distributed/distributed-from-config-file.sh
|
||||
|
||||
test-upgrade: install-race
|
||||
@echo "Running minio upgrade tests"
|
||||
@(env bash $(PWD)/buildscripts/minio-upgrade.sh)
|
||||
|
||||
@@ -86,18 +92,18 @@ test-replication-3site:
|
||||
test-delete-replication:
|
||||
@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
|
||||
|
||||
test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
|
||||
test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
|
||||
@echo "Running tests for replicating three sites"
|
||||
|
||||
test-site-replication-ldap: install ## verify automatic site replication
|
||||
test-site-replication-ldap: install-race ## verify automatic site replication
|
||||
@echo "Running tests for automatic site replication of IAM (with LDAP)"
|
||||
@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
|
||||
|
||||
test-site-replication-oidc: install ## verify automatic site replication
|
||||
test-site-replication-oidc: install-race ## verify automatic site replication
|
||||
@echo "Running tests for automatic site replication of IAM (with OIDC)"
|
||||
@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
|
||||
|
||||
test-site-replication-minio: install ## verify automatic site replication
|
||||
test-site-replication-minio: install-race ## verify automatic site replication
|
||||
@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
|
||||
@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
|
||||
|
||||
@@ -128,6 +134,9 @@ verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
|
||||
|
||||
build-debugging:
|
||||
@(env bash $(PWD)/docs/debugging/build.sh)
|
||||
|
||||
build: checks ## builds minio to $(PWD)
|
||||
@echo "Building minio binary to './minio'"
|
||||
@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@@ -138,14 +147,23 @@ hotfix-vars:
|
||||
$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
|
||||
$(eval TAG := "minio/minio:$(VERSION)")
|
||||
|
||||
hotfix: hotfix-vars install ## builds minio binary with hotfix tags
|
||||
@mv -f ./minio ./minio.$(VERSION)
|
||||
@minisign -qQSm ./minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
|
||||
@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum
|
||||
hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
|
||||
@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.0/pkger_2.2.0_linux_amd64.deb
|
||||
@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.0/linux-systemd/distributed/minio.service
|
||||
@sudo apt install ./pkger_2.2.0_linux_amd64.deb --yes
|
||||
@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
|
||||
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio
|
||||
@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)
|
||||
@minisign -qQSm minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
|
||||
@sha256sum < minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION).sha256sum
|
||||
@cp -af minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)* minio-release/$(GOOS)-$(GOARCH)/archive/
|
||||
@pkger -r $(VERSION) --ignore
|
||||
|
||||
hotfix-push: hotfix
|
||||
@scp -q -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
|
||||
@scp -q -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/
|
||||
@scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive
|
||||
@echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.$(VERSION)"
|
||||
|
||||
docker-hotfix-push: docker-hotfix
|
||||
@@ -159,6 +177,12 @@ docker: build ## builds minio docker container
|
||||
@echo "Building minio docker image '$(TAG)'"
|
||||
@docker build -q --no-cache -t $(TAG) . -f Dockerfile
|
||||
|
||||
install-race: checks ## builds minio to $(PWD)
|
||||
@echo "Building minio binary to './minio'"
|
||||
@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
|
||||
@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
|
||||
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
|
||||
|
||||
install: build ## builds minio and installs it to $GOPATH/bin.
|
||||
@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
|
||||
@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
|
||||
@@ -174,3 +198,6 @@ clean: ## cleanup all generated assets
|
||||
@rm -rvf build
|
||||
@rm -rvf release
|
||||
@rm -rvf .verify*
|
||||
@rm -rvf minio-release
|
||||
@rm -rvf minio.RELEASE*.hotfix.*
|
||||
@rm -rvf pkger_*.deb
|
||||
|
||||
@@ -259,6 +259,6 @@ Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/ma
|
||||
|
||||
## License
|
||||
|
||||
- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
|
||||
- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
|
||||
- MinIO source is licensed under the [GNU AGPLv3](https://github.com/minio/minio/blob/master/LICENSE).
|
||||
- MinIO [documentation](https://github.com/minio/minio/tree/master/docs) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
|
||||
- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)
|
||||
|
||||
@@ -9,7 +9,7 @@ function _init() {
|
||||
export CGO_ENABLED=0
|
||||
|
||||
## List of architectures and OS to test coss compilation.
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips"
|
||||
SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
|
||||
}
|
||||
|
||||
function _build() {
|
||||
|
||||
@@ -56,6 +56,8 @@ done
|
||||
|
||||
set +e
|
||||
|
||||
sleep 10
|
||||
|
||||
./mc ls minioadm/
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "listing failed, 'minioadmin' should be enabled"
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
set -E
|
||||
set -o pipefail
|
||||
set -x
|
||||
set -e
|
||||
|
||||
WORK_DIR="$PWD/.verify-$RANDOM"
|
||||
MINIO_CONFIG_DIR="$WORK_DIR/.minio"
|
||||
|
||||
@@ -87,8 +87,8 @@ function verify_rewrite() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go build ./docs/debugging/s3-check-md5/
|
||||
if ! ./s3-check-md5 \
|
||||
go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key minio \
|
||||
@@ -113,7 +113,7 @@ function verify_rewrite() {
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
|
||||
sleep 1
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key minio \
|
||||
|
||||
@@ -82,8 +82,8 @@ function start_minio_16drive() {
|
||||
rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
|
||||
sleep 10
|
||||
|
||||
go build ./docs/debugging/s3-check-md5/
|
||||
if ! ./s3-check-md5 \
|
||||
go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
@@ -111,7 +111,7 @@ function start_minio_16drive() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
@@ -134,7 +134,7 @@ function start_minio_16drive() {
|
||||
|
||||
"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
|
||||
@@ -9,11 +9,6 @@ x-minio-common: &minio-common
|
||||
expose:
|
||||
- "9000"
|
||||
- "9001"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
+1
-1
@@ -25,7 +25,7 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// Data types used for returning dummy access control
|
||||
|
||||
+103
-181
@@ -41,8 +41,7 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -58,7 +57,7 @@ const (
|
||||
func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketQuotaAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -95,7 +94,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
Quota: data,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if quotaConfig.Quota == 0 {
|
||||
if quotaConfig.Size == 0 && quotaConfig.Quota == 0 {
|
||||
bucketMeta.Quota = nil
|
||||
}
|
||||
|
||||
@@ -110,7 +109,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
|
||||
func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketQuotaAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -148,7 +147,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
|
||||
update := r.Form.Get("update") == "true"
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -291,7 +290,7 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
|
||||
arnType := vars["type"]
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketTargetAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -325,7 +324,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
|
||||
arn := vars["arn"]
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketTargetAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -365,7 +364,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
|
||||
bucket := pathClean(r.Form.Get("bucket"))
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportBucketMetadataAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportBucketMetadataAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -393,7 +392,8 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
// of bucket metadata
|
||||
zipWriter := zip.NewWriter(w)
|
||||
defer zipWriter.Close()
|
||||
rawDataFn := func(r io.Reader, filename string, sz int) error {
|
||||
|
||||
rawDataFn := func(r io.Reader, filename string, sz int) {
|
||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||
name: filename,
|
||||
size: int64(sz),
|
||||
@@ -402,20 +402,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
isDir: false,
|
||||
sys: nil,
|
||||
})
|
||||
if zerr != nil {
|
||||
logger.LogIf(ctx, zerr)
|
||||
return nil
|
||||
if zerr == nil {
|
||||
header.Method = zip.Deflate
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
if zerr == nil {
|
||||
io.Copy(zwriter, r)
|
||||
}
|
||||
}
|
||||
header.Method = zip.Deflate
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
if zerr != nil {
|
||||
logger.LogIf(ctx, zerr)
|
||||
return nil
|
||||
}
|
||||
if _, err := io.Copy(zwriter, r); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
cfgFiles := []string{
|
||||
@@ -447,10 +440,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketLifecycleConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -466,10 +456,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketQuotaConfigFile:
|
||||
config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
|
||||
if err != nil {
|
||||
@@ -484,10 +471,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketSSEConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -502,10 +486,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketTaggingConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -520,10 +501,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case objectLockConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -539,10 +517,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketVersioningConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -558,10 +533,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketReplicationConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
|
||||
if err != nil {
|
||||
@@ -576,11 +548,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketTargetsFile:
|
||||
config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -596,10 +564,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -647,7 +612,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportBucketMetadataAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -662,12 +627,31 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
|
||||
return
|
||||
}
|
||||
bucketMap := make(map[string]struct{}, 1)
|
||||
rpt := importMetaReport{
|
||||
madmin.BucketMetaImportErrs{
|
||||
Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
|
||||
},
|
||||
}
|
||||
|
||||
bucketMap := make(map[string]*BucketMetadata, len(zr.File))
|
||||
|
||||
updatedAt := UTCNow()
|
||||
|
||||
for _, file := range zr.File {
|
||||
slc := strings.Split(file.Name, slashSeparator)
|
||||
if len(slc) != 2 { // expecting bucket/configfile in the zipfile
|
||||
rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
|
||||
continue
|
||||
}
|
||||
bucket := slc[0]
|
||||
meta, err := readBucketMetadata(ctx, objectAPI, bucket)
|
||||
if err == nil {
|
||||
bucketMap[bucket] = &meta
|
||||
} else if err != errConfigNotFound {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
}
|
||||
}
|
||||
|
||||
// import object lock config if any - order of import matters here.
|
||||
for _, file := range zr.File {
|
||||
slc := strings.Split(file.Name, slashSeparator)
|
||||
@@ -704,7 +688,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
}
|
||||
bucketMap[bucket] = struct{}{}
|
||||
v := newBucketMetadata(bucket)
|
||||
bucketMap[bucket] = &v
|
||||
}
|
||||
|
||||
// Deny object locking configuration settings on existing buckets without object lock enabled.
|
||||
@@ -713,27 +698,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
bucketMap[bucket].ObjectLockConfigXML = configData
|
||||
bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeObjectLockConfig,
|
||||
Bucket: bucket,
|
||||
ObjectLockConfig: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -763,7 +730,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
}
|
||||
bucketMap[bucket] = struct{}{}
|
||||
v := newBucketMetadata(bucket)
|
||||
bucketMap[bucket] = &v
|
||||
}
|
||||
|
||||
if globalSiteReplicationSys.isEnabled() && v.Suspended() {
|
||||
@@ -786,10 +754,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
bucketMap[bucket].VersioningConfigXML = configData
|
||||
bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
}
|
||||
}
|
||||
@@ -807,6 +773,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
bucket, fileName := slc[0], slc[1]
|
||||
|
||||
// create bucket if it does not exist yet.
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
|
||||
@@ -816,7 +783,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
}
|
||||
bucketMap[bucket] = struct{}{}
|
||||
v := newBucketMetadata(bucket)
|
||||
bucketMap[bucket] = &v
|
||||
}
|
||||
if _, ok := bucketMap[bucket]; !ok {
|
||||
continue
|
||||
@@ -835,12 +803,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
rulesMap := config.ToRulesMap()
|
||||
globalEventNotifier.AddRulesMap(bucket, rulesMap)
|
||||
bucketMap[bucket].NotificationConfigXML = configData
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketPolicyConfig:
|
||||
// Error out if Content-Length is beyond allowed size.
|
||||
@@ -855,7 +818,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
||||
bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
@@ -873,22 +836,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
bucketMap[bucket].PolicyConfigJSON = configData
|
||||
bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypePolicy,
|
||||
Bucket: bucket,
|
||||
Policy: bucketPolicyBytes,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
case bucketLifecycleConfig:
|
||||
bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
|
||||
if err != nil {
|
||||
@@ -914,10 +864,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
bucketMap[bucket].LifecycleConfigXML = configData
|
||||
bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
case bucketSSEConfig:
|
||||
// Parse bucket encryption xml
|
||||
@@ -952,29 +900,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
// Store the bucket encryption configuration in the object layer
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
bucketMap[bucket].EncryptionConfigXML = configData
|
||||
bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeSSEConfig,
|
||||
Bucket: bucket,
|
||||
SSEConfig: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
case bucketTaggingConfig:
|
||||
tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
|
||||
if err != nil {
|
||||
@@ -988,27 +916,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
bucketMap[bucket].TaggingConfigXML = configData
|
||||
bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
// Call site replication hook.
|
||||
//
|
||||
// We encode the xml bytes as base64 to ensure there are no encoding
|
||||
// errors.
|
||||
cfgStr := base64.StdEncoding.EncodeToString(configData)
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeTags,
|
||||
Bucket: bucket,
|
||||
Tags: &cfgStr,
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
case bucketQuotaConfigFile:
|
||||
data, err := io.ReadAll(reader)
|
||||
if err != nil {
|
||||
@@ -1016,37 +926,49 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
continue
|
||||
}
|
||||
|
||||
quotaConfig, err := parseBucketQuota(bucket, data)
|
||||
_, err = parseBucketQuota(bucket, data)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
|
||||
updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
bucketMap[bucket].QuotaConfigJSON = data
|
||||
bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
|
||||
rpt.SetStatus(bucket, fileName, nil)
|
||||
|
||||
bucketMeta := madmin.SRBucketMeta{
|
||||
Type: madmin.SRBucketMetaTypeQuotaConfig,
|
||||
Bucket: bucket,
|
||||
Quota: data,
|
||||
UpdatedAt: updatedAt,
|
||||
}
|
||||
if quotaConfig.Quota == 0 {
|
||||
bucketMeta.Quota = nil
|
||||
}
|
||||
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
|
||||
rpt.SetStatus(bucket, fileName, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
enc := func(b []byte) *string {
|
||||
if b == nil {
|
||||
return nil
|
||||
}
|
||||
v := base64.StdEncoding.EncodeToString(b)
|
||||
return &v
|
||||
}
|
||||
|
||||
for bucket, meta := range bucketMap {
|
||||
err := globalBucketMetadataSys.save(ctx, *meta)
|
||||
if err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
// Call site replication hook.
|
||||
if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
|
||||
Bucket: bucket,
|
||||
Quota: meta.QuotaConfigJSON,
|
||||
Policy: meta.PolicyConfigJSON,
|
||||
Versioning: enc(meta.VersioningConfigXML),
|
||||
Tags: enc(meta.TaggingConfigXML),
|
||||
ObjectLockConfig: enc(meta.ObjectLockConfigXML),
|
||||
SSEConfig: enc(meta.EncryptionConfigXML),
|
||||
UpdatedAt: updatedAt,
|
||||
}); err != nil {
|
||||
rpt.SetStatus(bucket, "", err)
|
||||
continue
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -1064,7 +986,7 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1126,7 +1048,7 @@ func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.R
|
||||
vars := mux.Vars(r)
|
||||
bucket := vars["bucket"]
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -27,14 +27,14 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// validateAdminReq will validate request against and return whether it is allowed.
|
||||
// If any of the supplied actions are allowed it will be successful.
|
||||
// If nil ObjectLayer is returned, the operation is not permitted.
|
||||
// When nil ObjectLayer has been returned an error has always been sent to w.
|
||||
func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
|
||||
func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...policy.AdminAction) (ObjectLayer, auth.Credentials) {
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
@@ -78,7 +78,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
|
||||
|
||||
var apiErr APIError
|
||||
switch e := err.(type) {
|
||||
case iampolicy.Error:
|
||||
case policy.Error:
|
||||
apiErr = APIError{
|
||||
Code: "XMinioMalformedIAMPolicy",
|
||||
Description: e.Error(),
|
||||
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
"github.com/minio/minio/internal/config/etcd"
|
||||
xldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
@@ -38,14 +37,14 @@ import (
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
|
||||
func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -149,7 +148,7 @@ type setConfigResult struct {
|
||||
func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -242,7 +241,7 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
|
||||
func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -288,7 +287,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -321,7 +320,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
|
||||
func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -367,7 +366,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
|
||||
func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -405,7 +404,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
|
||||
func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -430,7 +429,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
|
||||
func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -482,7 +481,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -500,8 +499,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
switch hkv.Key {
|
||||
case config.EtcdSubSys:
|
||||
off = !etcd.Enabled(item.Config)
|
||||
case config.CacheSubSys:
|
||||
off = !cache.Enabled(item.Config)
|
||||
case config.StorageClassSubSys:
|
||||
off = !storageclass.Enabled(item.Config)
|
||||
case config.PolicyPluginSubSys:
|
||||
|
||||
@@ -33,12 +33,12 @@ import (
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/ldap"
|
||||
"github.com/minio/pkg/v2/ldap"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -121,7 +121,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
|
||||
// IDP config is not dynamic. Sanity check.
|
||||
if dynamic {
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), "", r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -220,7 +220,7 @@ func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *ht
|
||||
func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -273,7 +273,7 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
|
||||
func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -332,7 +332,7 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
|
||||
func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -19,13 +19,17 @@ package cmd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
|
||||
@@ -50,7 +54,7 @@ func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r
|
||||
// Check authorization.
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r,
|
||||
iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
|
||||
policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -96,7 +100,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
|
||||
|
||||
// Check authorization.
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -175,3 +179,267 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
|
||||
// AddServiceAccountLDAP adds a new service account for provided LDAP username or DN
|
||||
//
|
||||
// PUT /minio/admin/v3/idp/ldap/add-service-account
|
||||
func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
|
||||
if APIError.Code != "" {
|
||||
writeErrorResponseJSON(ctx, w, APIError, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// fail if ldap is not enabled
|
||||
if !globalIAMSys.LDAPConfig.Enabled() {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("LDAP not enabled")), r.URL)
|
||||
}
|
||||
|
||||
// Find the user for the request sender (as it may be sent via a service
|
||||
// account or STS account):
|
||||
requestorUser := cred.AccessKey
|
||||
requestorParentUser := cred.AccessKey
|
||||
requestorGroups := cred.Groups
|
||||
requestorIsDerivedCredential := false
|
||||
if cred.IsServiceAccount() || cred.IsTemp() {
|
||||
requestorParentUser = cred.ParentUser
|
||||
requestorIsDerivedCredential = true
|
||||
}
|
||||
|
||||
// Check if we are creating svc account for request sender.
|
||||
isSvcAccForRequestor := false
|
||||
if targetUser == requestorUser || targetUser == requestorParentUser {
|
||||
isSvcAccForRequestor = true
|
||||
}
|
||||
|
||||
var (
|
||||
targetGroups []string
|
||||
err error
|
||||
)
|
||||
|
||||
// If we are creating svc account for request sender, ensure
|
||||
// that targetUser is a real user (i.e. not derived
|
||||
// credentials).
|
||||
if isSvcAccForRequestor {
|
||||
if requestorIsDerivedCredential {
|
||||
if requestorParentUser == "" {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
|
||||
errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL)
|
||||
return
|
||||
}
|
||||
targetUser = requestorParentUser
|
||||
}
|
||||
targetGroups = requestorGroups
|
||||
|
||||
// Deny if the target user is not LDAP
|
||||
isLDAP, err := globalIAMSys.LDAPConfig.DoesUsernameExist(targetUser)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if isLDAP == "" {
|
||||
err := errors.New("Specified user does not exist on LDAP server")
|
||||
APIErr := errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err)
|
||||
writeErrorResponseJSON(ctx, w, APIErr, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// In case of LDAP/OIDC we need to set `opts.claims` to ensure
|
||||
// it is associated with the LDAP/OIDC user properly.
|
||||
for k, v := range cred.Claims {
|
||||
if k == expClaim {
|
||||
continue
|
||||
}
|
||||
opts.claims[k] = v
|
||||
}
|
||||
} else {
|
||||
isDN := globalIAMSys.LDAPConfig.IsLDAPUserDN(targetUser)
|
||||
|
||||
opts.claims[ldapUserN] = targetUser // simple username
|
||||
targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
|
||||
if err != nil {
|
||||
// if not found, check if DN
|
||||
if strings.Contains(err.Error(), "not found") && isDN {
|
||||
// warn user that DNs are not allowed
|
||||
err = fmt.Errorf("Must use short username to add service account. %w", err)
|
||||
}
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
opts.claims[ldapUser] = targetUser // DN
|
||||
}
|
||||
|
||||
newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
createResp := madmin.AddServiceAccountResp{
|
||||
Credentials: madmin.Credentials{
|
||||
AccessKey: newCred.AccessKey,
|
||||
SecretKey: newCred.SecretKey,
|
||||
Expiration: newCred.Expiration,
|
||||
},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(createResp)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
|
||||
// Call hook for cluster-replication if the service account is not for a
|
||||
// root user.
|
||||
if newCred.ParentUser != globalActiveCred.AccessKey {
|
||||
logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
|
||||
Type: madmin.SRIAMItemSvcAcc,
|
||||
SvcAccChange: &madmin.SRSvcAccChange{
|
||||
Create: &madmin.SRSvcAccCreate{
|
||||
Parent: newCred.ParentUser,
|
||||
AccessKey: newCred.AccessKey,
|
||||
SecretKey: newCred.SecretKey,
|
||||
Groups: newCred.Groups,
|
||||
Name: newCred.Name,
|
||||
Description: newCred.Description,
|
||||
Claims: opts.claims,
|
||||
SessionPolicy: createReq.Policy,
|
||||
Status: auth.AccountOn,
|
||||
Expiration: createReq.Expiration,
|
||||
},
|
||||
},
|
||||
UpdatedAt: updatedAt,
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
// ListAccessKeysLDAP - GET /minio/admin/v3/idp/ldap/list-access-keys
|
||||
func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
userDN := r.Form.Get("userDN")
|
||||
|
||||
// If listing is requested for a specific user (who is not the request
|
||||
// sender), check that the user has permissions.
|
||||
if userDN != "" && userDN != cred.ParentUser {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
userDN = cred.AccessKey
|
||||
if cred.ParentUser != "" {
|
||||
userDN = cred.ParentUser
|
||||
}
|
||||
}
|
||||
|
||||
targetAccount, err := globalIAMSys.LDAPConfig.DoesUsernameExist(userDN)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
} else if userDN == "" {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
listType := r.Form.Get("listType")
|
||||
if listType != "sts-only" && listType != "svcacc-only" && listType != "" {
|
||||
// default to both
|
||||
listType = ""
|
||||
}
|
||||
|
||||
var serviceAccounts []auth.Credentials
|
||||
var stsKeys []auth.Credentials
|
||||
|
||||
if listType == "" || listType == "sts-only" {
|
||||
stsKeys, err = globalIAMSys.ListSTSAccounts(ctx, targetAccount)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
if listType == "" || listType == "svcacc-only" {
|
||||
serviceAccounts, err = globalIAMSys.ListServiceAccounts(ctx, targetAccount)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
var serviceAccountList []madmin.ServiceAccountInfo
|
||||
var stsKeyList []madmin.ServiceAccountInfo
|
||||
|
||||
for _, svc := range serviceAccounts {
|
||||
expiryTime := svc.Expiration
|
||||
serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
|
||||
AccessKey: svc.AccessKey,
|
||||
Expiration: &expiryTime,
|
||||
})
|
||||
}
|
||||
for _, sts := range stsKeys {
|
||||
expiryTime := sts.Expiration
|
||||
stsKeyList = append(stsKeyList, madmin.ServiceAccountInfo{
|
||||
AccessKey: sts.AccessKey,
|
||||
Expiration: &expiryTime,
|
||||
})
|
||||
}
|
||||
|
||||
listResp := madmin.ListAccessKeysLDAPResp{
|
||||
ServiceAccounts: serviceAccountList,
|
||||
STSKeys: stsKeyList,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(listResp)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
writeSuccessResponseJSON(w, encryptedData)
|
||||
}
|
||||
|
||||
+46
-15
@@ -22,11 +22,12 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -37,7 +38,7 @@ var (
|
||||
func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -66,16 +67,28 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
||||
|
||||
vars := mux.Vars(r)
|
||||
v := vars["pool"]
|
||||
byID := vars["by-id"] == "true"
|
||||
|
||||
pools := strings.Split(v, ",")
|
||||
poolIndices := make([]int, 0, len(pools))
|
||||
|
||||
for _, pool := range pools {
|
||||
idx := globalEndpoints.GetPoolIdx(pool)
|
||||
if idx == -1 {
|
||||
// We didn't find any matching pools, invalid input
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
return
|
||||
var idx int
|
||||
if byID {
|
||||
var err error
|
||||
idx, err = strconv.Atoi(pool)
|
||||
if err != nil {
|
||||
// We didn't find any matching pools, invalid input
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
} else {
|
||||
idx = globalEndpoints.GetPoolIdx(pool)
|
||||
if idx == -1 {
|
||||
// We didn't find any matching pools, invalid input
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
var pool *erasureSets
|
||||
for pidx := range z.serverPools {
|
||||
@@ -113,7 +126,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
|
||||
func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -132,8 +145,17 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
|
||||
|
||||
vars := mux.Vars(r)
|
||||
v := vars["pool"]
|
||||
byID := vars["by-id"] == "true"
|
||||
idx := -1
|
||||
|
||||
if byID {
|
||||
if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
|
||||
idx = i
|
||||
}
|
||||
} else {
|
||||
idx = globalEndpoints.GetPoolIdx(v)
|
||||
}
|
||||
|
||||
idx := globalEndpoints.GetPoolIdx(v)
|
||||
if idx == -1 {
|
||||
// We didn't find any matching pools, invalid input
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
|
||||
@@ -159,7 +181,7 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -178,8 +200,17 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
vars := mux.Vars(r)
|
||||
v := vars["pool"]
|
||||
byID := vars["by-id"] == "true"
|
||||
idx := -1
|
||||
|
||||
if byID {
|
||||
if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) {
|
||||
idx = i
|
||||
}
|
||||
} else {
|
||||
idx = globalEndpoints.GetPoolIdx(v)
|
||||
}
|
||||
|
||||
idx := globalEndpoints.GetPoolIdx(v)
|
||||
if idx == -1 {
|
||||
apiErr := toAdminAPIErr(ctx, errInvalidArgument)
|
||||
apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v)
|
||||
@@ -200,7 +231,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -233,7 +264,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -304,7 +335,7 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
|
||||
func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -344,7 +375,7 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
|
||||
func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
@@ -31,17 +31,17 @@ import (
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
|
||||
func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -52,7 +52,8 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
|
||||
opts := getSRAddOptions(r)
|
||||
status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -68,6 +69,11 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
|
||||
writeSuccessResponseJSON(w, body)
|
||||
}
|
||||
|
||||
func getSRAddOptions(r *http.Request) (opts madmin.SRAddOptions) {
|
||||
opts.ReplicateILMExpiry = r.Form.Get("replicateILMExpiry") == "true"
|
||||
return
|
||||
}
|
||||
|
||||
// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
|
||||
//
|
||||
// used internally to tell current cluster to enable SR with
|
||||
@@ -75,7 +81,7 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -97,7 +103,7 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -144,7 +150,7 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
|
||||
func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -163,7 +169,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
|
||||
if item.Policy == nil {
|
||||
err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt)
|
||||
} else {
|
||||
policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
|
||||
policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy))
|
||||
if perr != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
|
||||
return
|
||||
@@ -192,11 +198,11 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
|
||||
}
|
||||
}
|
||||
|
||||
// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
|
||||
// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/peer/bucket-meta
|
||||
func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -207,15 +213,20 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
if item.Bucket == "" {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var err error
|
||||
switch item.Type {
|
||||
default:
|
||||
err = errSRInvalidRequest(errInvalidArgument)
|
||||
err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
|
||||
case madmin.SRBucketMetaTypePolicy:
|
||||
if item.Policy == nil {
|
||||
err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
|
||||
} else {
|
||||
bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
|
||||
bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket)
|
||||
if berr != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
|
||||
return
|
||||
@@ -236,7 +247,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
@@ -248,6 +259,8 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaTypeSSEConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt)
|
||||
case madmin.SRBucketMetaLCConfig:
|
||||
err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt)
|
||||
}
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
@@ -260,7 +273,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
|
||||
func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -280,7 +293,7 @@ func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Req
|
||||
func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -317,7 +330,7 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
|
||||
func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -329,6 +342,7 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
|
||||
opts.Users = true
|
||||
opts.Policies = true
|
||||
opts.Groups = true
|
||||
opts.ILMExpiryRules = true
|
||||
}
|
||||
info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts)
|
||||
if err != nil {
|
||||
@@ -346,7 +360,7 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
|
||||
func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -368,7 +382,7 @@ func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http
|
||||
func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -378,7 +392,9 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site)
|
||||
|
||||
opts := getSREditOptions(r)
|
||||
status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -393,13 +409,19 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
|
||||
writeSuccessResponseJSON(w, body)
|
||||
}
|
||||
|
||||
func getSREditOptions(r *http.Request) (opts madmin.SREditOptions) {
|
||||
opts.DisableILMExpiryReplication = r.Form.Get("disableILMExpiryReplication") == "true"
|
||||
opts.EnableILMExpiryReplication = r.Form.Get("enableILMExpiryReplication") == "true"
|
||||
return
|
||||
}
|
||||
|
||||
// SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit
|
||||
//
|
||||
// used internally to tell current cluster to update endpoint for peer
|
||||
func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -417,15 +439,41 @@ func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
|
||||
// SRStateEdit - PUT /minio/admin/v3/site-replication/state/edit
|
||||
//
|
||||
// used internally to tell current cluster to update site replication state
|
||||
func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
var state madmin.SRStateEditReq
|
||||
if err := parseJSONBody(ctx, r.Body, &state, ""); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
|
||||
q := r.Form
|
||||
opts.Buckets = q.Get("buckets") == "true"
|
||||
opts.Policies = q.Get("policies") == "true"
|
||||
opts.Groups = q.Get("groups") == "true"
|
||||
opts.Users = q.Get("users") == "true"
|
||||
opts.ILMExpiryRules = q.Get("ilm-expiry-rules") == "true"
|
||||
opts.PeerState = q.Get("peer-state") == "true"
|
||||
opts.Entity = madmin.GetSREntityType(q.Get("entity"))
|
||||
opts.EntityValue = q.Get("entityvalue")
|
||||
opts.ShowDeleted = q.Get("showDeleted") == "true"
|
||||
opts.Metrics = q.Get("metrics") == "true"
|
||||
return
|
||||
}
|
||||
|
||||
@@ -433,7 +481,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
|
||||
func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -464,7 +512,7 @@ func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.R
|
||||
func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -486,7 +534,7 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationResyncAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationResyncAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -532,7 +580,7 @@ func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.
|
||||
|
||||
connectTime := time.Now()
|
||||
for {
|
||||
n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
|
||||
n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
|
||||
atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
|
||||
|
||||
@@ -32,7 +32,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
minio "github.com/minio/minio-go/v7"
|
||||
"github.com/minio/pkg/sync/errgroup"
|
||||
"github.com/minio/pkg/v2/sync/errgroup"
|
||||
)
|
||||
|
||||
func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
|
||||
|
||||
+206
-186
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -34,14 +35,14 @@ import (
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
|
||||
func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.DeleteUserAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -85,7 +86,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -119,7 +120,7 @@ func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request
|
||||
func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -186,10 +187,10 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
|
||||
checkDenyOnly = true
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.GetUserAdminAction,
|
||||
Action: policy.GetUserAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -218,7 +219,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.AddUserToGroupAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -288,7 +289,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetGroupAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetGroupAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -315,7 +316,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListGroupsAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -339,7 +340,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -382,7 +383,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
|
||||
func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
|
||||
objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -470,10 +471,10 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
|
||||
checkDenyOnly = true
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.CreateUserAdminAction,
|
||||
Action: policy.CreateUserAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -544,34 +545,41 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
args := policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListTemporaryAccountsAdminAction,
|
||||
Action: policy.ListTemporaryAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(args) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
stsAccount, policy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
|
||||
stsAccount, sessionPolicy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var stsAccountPolicy iampolicy.Policy
|
||||
var stsAccountPolicy policy.Policy
|
||||
|
||||
if policy != nil {
|
||||
stsAccountPolicy = *policy
|
||||
if sessionPolicy != nil {
|
||||
stsAccountPolicy = *sessionPolicy
|
||||
} else {
|
||||
policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, false)
|
||||
policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, stsAccount.Groups...)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if len(policiesNames) == 0 {
|
||||
policySet, _ := args.GetPolicies(iamPolicyClaimNameOpenID())
|
||||
policiesNames = policySet.ToSlice()
|
||||
}
|
||||
|
||||
stsAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...)
|
||||
}
|
||||
|
||||
@@ -584,7 +592,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
|
||||
infoResp := madmin.TemporaryAccountInfoResp{
|
||||
ParentUser: stsAccount.ParentUser,
|
||||
AccountStatus: stsAccount.Status,
|
||||
ImpliedPolicy: policy == nil,
|
||||
ImpliedPolicy: sessionPolicy == nil,
|
||||
Policy: string(policyJSON),
|
||||
Expiration: &stsAccount.Expiration,
|
||||
}
|
||||
@@ -606,72 +614,17 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// AddServiceAccount - PUT /minio/admin/v3/add-service-account
|
||||
func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
password := cred.SecretKey
|
||||
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var createReq madmin.AddServiceAccountReq
|
||||
if err = json.Unmarshal(reqBytes, &createReq); err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
// service account access key cannot have space characters beginning and end of the string.
|
||||
if hasSpaceBE(createReq.AccessKey) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err := createReq.Validate(); err != nil {
|
||||
// Since this validation would happen client side as well, we only send
|
||||
// a generic error message here.
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
|
||||
ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r)
|
||||
if APIError.Code != "" {
|
||||
writeErrorResponseJSON(ctx, w, APIError, r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
var (
|
||||
targetUser string
|
||||
targetGroups []string
|
||||
err error
|
||||
)
|
||||
|
||||
// If the request did not set a TargetUser, the service account is
|
||||
// created for the request sender.
|
||||
targetUser = createReq.TargetUser
|
||||
if targetUser == "" {
|
||||
targetUser = cred.AccessKey
|
||||
}
|
||||
|
||||
description := createReq.Description
|
||||
if description == "" {
|
||||
description = createReq.Comment
|
||||
}
|
||||
opts := newServiceAccountOpts{
|
||||
accessKey: createReq.AccessKey,
|
||||
secretKey: createReq.SecretKey,
|
||||
name: createReq.Name,
|
||||
description: description,
|
||||
expiration: createReq.Expiration,
|
||||
claims: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Find the user for the request sender (as it may be sent via a service
|
||||
// account or STS account):
|
||||
requestorUser := cred.AccessKey
|
||||
@@ -705,23 +658,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
// that targetUser is a real user (i.e. not derived
|
||||
// credentials).
|
||||
if isSvcAccForRequestor {
|
||||
// Check if adding service account is explicitly denied.
|
||||
//
|
||||
// This allows turning off service accounts for request sender,
|
||||
// if there is no deny statement this call is implicitly enabled.
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: requestorUser,
|
||||
Groups: requestorGroups,
|
||||
Action: iampolicy.CreateServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: true,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if requestorIsDerivedCredential {
|
||||
if requestorParentUser == "" {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx,
|
||||
@@ -740,32 +676,16 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
}
|
||||
opts.claims[k] = v
|
||||
}
|
||||
} else {
|
||||
// Need permission if we are creating a service account for a
|
||||
// user <> to the request sender
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
AccountName: requestorUser,
|
||||
Groups: requestorGroups,
|
||||
Action: iampolicy.CreateServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
}) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
} else if globalIAMSys.LDAPConfig.Enabled() {
|
||||
// In case of LDAP we need to resolve the targetUser to a DN and
|
||||
// query their groups:
|
||||
if globalIAMSys.LDAPConfig.Enabled() {
|
||||
opts.claims[ldapUserN] = targetUser // simple username
|
||||
targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
opts.claims[ldapUser] = targetUser // username DN
|
||||
opts.claims[ldapUserN] = targetUser // simple username
|
||||
targetUser, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
opts.claims[ldapUser] = targetUser // username DN
|
||||
|
||||
// NOTE: if not using LDAP, then internal IDP or open ID is
|
||||
// being used - in the former, group info is enforced when
|
||||
@@ -773,16 +693,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
// latter, a group notion is not supported.
|
||||
}
|
||||
|
||||
var sp *iampolicy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
sp, err = iampolicy.ParseConfig(bytes.NewReader(createReq.Policy))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
opts.sessionPolicy = sp
|
||||
newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -803,7 +713,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
return
|
||||
}
|
||||
|
||||
encryptedData, err := madmin.EncryptData(password, data)
|
||||
encryptedData, err := madmin.EncryptData(cred.SecretKey, data)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -864,10 +774,10 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.UpdateServiceAccountAdminAction,
|
||||
Action: policy.UpdateServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -903,13 +813,16 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
return
|
||||
}
|
||||
|
||||
var sp *iampolicy.Policy
|
||||
var sp *policy.Policy
|
||||
if len(updateReq.NewPolicy) > 0 {
|
||||
sp, err = iampolicy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if sp.Version == "" && len(sp.Statements) == 0 {
|
||||
sp = nil
|
||||
}
|
||||
}
|
||||
opts := updateServiceAccountOpts{
|
||||
secretKey: updateReq.NewSecretKey,
|
||||
@@ -970,16 +883,16 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
svcAccount, policy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
|
||||
svcAccount, sessionPolicy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListServiceAccountsAdminAction,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -995,12 +908,15 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
}
|
||||
|
||||
var svcAccountPolicy iampolicy.Policy
|
||||
// if session policy is nil or empty, then it is implied policy
|
||||
impliedPolicy := sessionPolicy == nil || (sessionPolicy.Version == "" && len(sessionPolicy.Statements) == 0)
|
||||
|
||||
if policy != nil {
|
||||
svcAccountPolicy = *policy
|
||||
var svcAccountPolicy policy.Policy
|
||||
|
||||
if !impliedPolicy {
|
||||
svcAccountPolicy = *sessionPolicy
|
||||
} else {
|
||||
policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, false)
|
||||
policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, svcAccount.Groups...)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1024,7 +940,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
Name: svcAccount.Name,
|
||||
Description: svcAccount.Description,
|
||||
AccountStatus: svcAccount.Status,
|
||||
ImpliedPolicy: policy == nil,
|
||||
ImpliedPolicy: impliedPolicy,
|
||||
Policy: string(policyJSON),
|
||||
Expiration: expiration,
|
||||
}
|
||||
@@ -1067,10 +983,10 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
|
||||
// sender), check that the user has permissions.
|
||||
user := r.Form.Get("user")
|
||||
if user != "" && user != cred.AccessKey {
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListServiceAccountsAdminAction,
|
||||
Action: policy.ListServiceAccountsAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -1146,12 +1062,16 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// We do not care if service account is readable or not at this point,
|
||||
// since this is a delete call we shall allow it to be deleted if possible.
|
||||
svcAccount, _, _ := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
|
||||
svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
|
||||
if errors.Is(err, errNoSuchServiceAccount) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.RemoveServiceAccountAdminAction,
|
||||
Action: policy.RemoveServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -1192,7 +1112,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeSuccessNoContent(w)
|
||||
}
|
||||
|
||||
// AccountInfoHandler returns usage
|
||||
// AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us
|
||||
func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
@@ -1219,10 +1139,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
enablePrefixUsage := r.Form.Get("prefix-usage") == "true"
|
||||
|
||||
isAllowedAccess := func(bucketName string) (rd, wr bool) {
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListBucketAction,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
@@ -1232,10 +1152,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
rd = true
|
||||
}
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.GetBucketLocationAction,
|
||||
Action: policy.GetBucketLocationAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
@@ -1245,10 +1165,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
rd = true
|
||||
}
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.PutObjectAction,
|
||||
Action: policy.PutObjectAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
@@ -1261,12 +1181,30 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return rd, wr
|
||||
}
|
||||
|
||||
// Load the latest calculated data usage
|
||||
dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
|
||||
bucketStorageCache.Once.Do(func() {
|
||||
// Set this to 10 secs since its enough, as scanner
|
||||
// does not update the bucket usage values frequently.
|
||||
bucketStorageCache.TTL = 10 * time.Second
|
||||
|
||||
// Rely on older value if usage loading fails from disk.
|
||||
bucketStorageCache.Relax = true
|
||||
bucketStorageCache.Update = func() (interface{}, error) {
|
||||
ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer done()
|
||||
|
||||
return loadDataUsageFromBackend(ctx, objectAPI)
|
||||
}
|
||||
})
|
||||
|
||||
var dataUsageInfo DataUsageInfo
|
||||
v, _ := bucketStorageCache.Get()
|
||||
if v != nil {
|
||||
dataUsageInfo, _ = v.(DataUsageInfo)
|
||||
}
|
||||
|
||||
// If etcd, dns federation configured list buckets from etcd.
|
||||
var buckets []BucketInfo
|
||||
var err error
|
||||
var buckets []BucketInfo
|
||||
if globalDNSConfig != nil && globalBucketFederation {
|
||||
dnsBuckets, err := globalDNSConfig.List()
|
||||
if err != nil && !IsErrIgnored(err,
|
||||
@@ -1285,7 +1223,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
return buckets[i].Name < buckets[j].Name
|
||||
})
|
||||
} else {
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
|
||||
buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1298,14 +1236,14 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
accountName = cred.ParentUser
|
||||
}
|
||||
|
||||
roleArn := iampolicy.Args{Claims: cred.Claims}.GetRoleArn()
|
||||
policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
|
||||
var effectivePolicy iampolicy.Policy
|
||||
roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn()
|
||||
policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
|
||||
var effectivePolicy policy.Policy
|
||||
|
||||
var buf []byte
|
||||
switch {
|
||||
case accountName == globalActiveCred.AccessKey:
|
||||
for _, policy := range iampolicy.DefaultPolicies {
|
||||
for _, policy := range policy.DefaultPolicies {
|
||||
if policy.Name == "consoleAdmin" {
|
||||
effectivePolicy = policy.Definition
|
||||
break
|
||||
@@ -1325,7 +1263,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...)
|
||||
|
||||
default:
|
||||
policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
|
||||
policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
@@ -1417,7 +1355,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetPolicyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1463,7 +1401,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
|
||||
func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1475,7 +1413,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
newPolicies := make(map[string]iampolicy.Policy)
|
||||
newPolicies := make(map[string]policy.Policy)
|
||||
for name, p := range policies {
|
||||
_, err = json.Marshal(p)
|
||||
if err != nil {
|
||||
@@ -1494,7 +1432,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1505,7 +1443,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
|
||||
return
|
||||
}
|
||||
|
||||
newPolicies := make(map[string]iampolicy.Policy)
|
||||
newPolicies := make(map[string]policy.Policy)
|
||||
for name, p := range policies {
|
||||
_, err = json.Marshal(p)
|
||||
if err != nil {
|
||||
@@ -1524,7 +1462,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.DeletePolicyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1550,7 +1488,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.CreatePolicyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1582,7 +1520,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
return
|
||||
}
|
||||
|
||||
iamPolicy, err := iampolicy.ParseConfig(bytes.NewReader(iamPolicyBytes))
|
||||
iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1614,7 +1552,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
|
||||
func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.AttachPolicyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1688,7 +1626,7 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
|
||||
|
||||
// Check authorization.
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r,
|
||||
iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
|
||||
policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1726,8 +1664,8 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
|
||||
func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction,
|
||||
iampolicy.AttachPolicyAdminAction)
|
||||
objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction,
|
||||
policy.AttachPolicyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1826,7 +1764,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportIAMAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -2096,7 +2034,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
default:
|
||||
defer f.Close()
|
||||
var allPolicies map[string]iampolicy.Policy
|
||||
var allPolicies map[string]policy.Policy
|
||||
data, err = io.ReadAll(f)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
|
||||
@@ -2177,10 +2115,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
checkDenyOnly = true
|
||||
}
|
||||
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.CreateUserAdminAction,
|
||||
Action: policy.CreateUserAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -2257,10 +2195,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
for user, svcAcctReq := range serviceAcctReqs {
|
||||
var sp *iampolicy.Policy
|
||||
var sp *policy.Policy
|
||||
var err error
|
||||
if len(svcAcctReq.SessionPolicy) > 0 {
|
||||
sp, err = iampolicy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
|
||||
return
|
||||
@@ -2271,10 +2209,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
|
||||
return
|
||||
}
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.CreateServiceAccountAdminAction,
|
||||
Action: policy.CreateServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -2475,3 +2413,85 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Get current object layer instance.
|
||||
objectAPI := newObjectLayerFn()
|
||||
if objectAPI == nil || globalNotificationSys == nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrServerNotInitialized)
|
||||
}
|
||||
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, "")
|
||||
if s3Err != ErrNone {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(s3Err)
|
||||
}
|
||||
|
||||
password := cred.SecretKey
|
||||
reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err)
|
||||
}
|
||||
|
||||
var createReq madmin.AddServiceAccountReq
|
||||
if err = json.Unmarshal(reqBytes, &createReq); err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err)
|
||||
}
|
||||
|
||||
// service account access key cannot have space characters beginning and end of the string.
|
||||
if hasSpaceBE(createReq.AccessKey) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument)
|
||||
}
|
||||
|
||||
if err := createReq.Validate(); err != nil {
|
||||
// Since this validation would happen client side as well, we only send
|
||||
// a generic error message here.
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument)
|
||||
}
|
||||
// If the request did not set a TargetUser, the service account is
|
||||
// created for the request sender.
|
||||
targetUser := createReq.TargetUser
|
||||
if targetUser == "" {
|
||||
targetUser = cred.AccessKey
|
||||
}
|
||||
|
||||
description := createReq.Description
|
||||
if description == "" {
|
||||
description = createReq.Comment
|
||||
}
|
||||
opts := newServiceAccountOpts{
|
||||
accessKey: createReq.AccessKey,
|
||||
secretKey: createReq.SecretKey,
|
||||
name: createReq.Name,
|
||||
description: description,
|
||||
expiration: createReq.Expiration,
|
||||
claims: make(map[string]interface{}),
|
||||
}
|
||||
|
||||
// Check if action is allowed if creating access key for another user
|
||||
// Check if action is explicitly denied if for self
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.CreateServiceAccountAdminAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
DenyOnly: (targetUser == cred.AccessKey || targetUser == cred.ParentUser),
|
||||
}) {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied)
|
||||
}
|
||||
|
||||
var sp *policy.Policy
|
||||
if len(createReq.Policy) > 0 {
|
||||
sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
|
||||
if err != nil {
|
||||
return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err)
|
||||
}
|
||||
}
|
||||
|
||||
opts.sessionPolicy = sp
|
||||
|
||||
return ctx, cred, opts, createReq, targetUser, APIError{}
|
||||
}
|
||||
|
||||
@@ -39,7 +39,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio-go/v7/pkg/signer"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/env"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+188
-60
@@ -41,6 +41,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
@@ -51,18 +52,22 @@ import (
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/logger/message/log"
|
||||
xnet "github.com/minio/pkg/net"
|
||||
"github.com/minio/pkg/v2/logger/message/log"
|
||||
xnet "github.com/minio/pkg/v2/net"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
"github.com/secure-io/sio-go"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
const (
|
||||
maxEConfigJSONSize = 262272
|
||||
kubernetesVersionEndpoint = "https://kubernetes.default.svc/version"
|
||||
anonymizeParam = "anonymize"
|
||||
anonymizeStrict = "strict"
|
||||
)
|
||||
|
||||
// Only valid query params for mgmt admin APIs.
|
||||
@@ -80,7 +85,7 @@ const (
|
||||
func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -251,11 +256,11 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
|
||||
var objectAPI ObjectLayer
|
||||
switch serviceSig {
|
||||
case serviceRestart:
|
||||
objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
|
||||
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
|
||||
case serviceStop:
|
||||
objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
|
||||
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
|
||||
case serviceFreeze, serviceUnFreeze:
|
||||
objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceFreezeAdminAction)
|
||||
objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
|
||||
}
|
||||
if objectAPI == nil {
|
||||
return
|
||||
@@ -330,12 +335,12 @@ type ServerHTTPStats struct {
|
||||
func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
storageInfo := objectAPI.StorageInfo(ctx)
|
||||
storageInfo := objectAPI.StorageInfo(ctx, true)
|
||||
|
||||
// Collect any disk healing.
|
||||
healing, _ := getAggregatedBackgroundHealState(ctx, nil)
|
||||
@@ -369,7 +374,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -478,7 +483,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
|
||||
func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.DataUsageInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -561,7 +566,7 @@ type PeerLocks struct {
|
||||
func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ForceUnlockAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -596,7 +601,7 @@ func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Requ
|
||||
func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.TopLocksAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -647,7 +652,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
|
||||
ctx := r.Context()
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -732,7 +737,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
|
||||
ctx := r.Context()
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -753,11 +758,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
}
|
||||
// read request body
|
||||
io.CopyN(io.Discard, r.Body, 1)
|
||||
|
||||
globalProfilerMu.Lock()
|
||||
|
||||
if globalProfiler == nil {
|
||||
globalProfiler = make(map[string]minioProfiler, 10)
|
||||
}
|
||||
@@ -827,7 +829,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
|
||||
ctx := r.Context()
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -925,7 +927,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
|
||||
func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1000,7 +1002,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
Message: hr.errBody,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
HostID: globalDeploymentID(),
|
||||
})
|
||||
}
|
||||
if !started {
|
||||
@@ -1024,7 +1026,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
|
||||
clientToken := nh.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s@%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
clientToken = fmt.Sprintf("%s:%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
ClientToken: clientToken,
|
||||
@@ -1055,7 +1057,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
respCh := make(chan healResp)
|
||||
respCh := make(chan healResp, 1)
|
||||
switch {
|
||||
case hip.forceStop:
|
||||
go func() {
|
||||
@@ -1109,7 +1111,7 @@ func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmi
|
||||
func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1130,7 +1132,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
|
||||
func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1146,6 +1148,7 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
|
||||
return
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer nsLock.Unlock(lkctx)
|
||||
|
||||
durationStr := r.Form.Get(peerRESTDuration)
|
||||
@@ -1172,11 +1175,71 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
|
||||
}
|
||||
}
|
||||
|
||||
// ClientDevNullExtraTime - return extratime for last devnull
|
||||
// [POST] /minio/admin/v3/speedtest/client/devnull/extratime
|
||||
func (a adminAPIHandlers) ClientDevNullExtraTime(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
if err := enc.Encode(madmin.ClientPerfExtraTime{TimeSpent: atomic.LoadInt64(&globalLastClientPerfExtraTime)}); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ClientDevNull - everything goes to io.Discard
|
||||
// [POST] /minio/admin/v3/speedtest/client/devnull
|
||||
func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
timeStart := time.Now()
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
|
||||
nsLock := objectAPI.NewNSLock(minioMetaBucket, "client-perf")
|
||||
lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
|
||||
return
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer nsLock.Unlock(lkctx)
|
||||
timeEnd := time.Now()
|
||||
|
||||
atomic.SwapInt64(&globalLastClientPerfExtraTime, timeEnd.Sub(timeStart).Nanoseconds())
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, madmin.MaxClientPerfTimeout)
|
||||
defer cancel()
|
||||
totalRx := int64(0)
|
||||
connectTime := time.Now()
|
||||
for {
|
||||
n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
// would mean the network is not stable. Logging here will help in debugging network issues.
|
||||
if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
}
|
||||
totalRx += n
|
||||
if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte {
|
||||
break
|
||||
}
|
||||
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
// NetperfHandler - perform mesh style network throughput test
|
||||
func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1192,6 +1255,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
|
||||
return
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer nsLock.Unlock(lkctx)
|
||||
|
||||
durationStr := r.Form.Get(peerRESTDuration)
|
||||
@@ -1221,7 +1285,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
|
||||
func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1233,6 +1297,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
|
||||
autotune := r.Form.Get("autotune") == "true"
|
||||
noClear := r.Form.Get("noclear") == "true"
|
||||
enableSha256 := r.Form.Get("enableSha256") == "true"
|
||||
|
||||
size, err := strconv.Atoi(sizeStr)
|
||||
if err != nil {
|
||||
@@ -1249,7 +1314,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
duration = time.Second * 10
|
||||
}
|
||||
|
||||
storageInfo := objectAPI.StorageInfo(ctx)
|
||||
storageInfo := objectAPI.StorageInfo(ctx, true)
|
||||
|
||||
sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
|
||||
if !sufficientCapacity {
|
||||
@@ -1302,6 +1367,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
autotune: autotune,
|
||||
storageClass: storageClass,
|
||||
bucketName: customBucket,
|
||||
enableSha256: enableSha256,
|
||||
})
|
||||
var prevResult madmin.SpeedTestResult
|
||||
for {
|
||||
@@ -1337,13 +1403,21 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
|
||||
if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
|
||||
if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{VersioningEnabled: globalSiteReplicationSys.isEnabled()}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
// Only BucketExists error can be ignored.
|
||||
return false, err
|
||||
}
|
||||
bucketExists = true
|
||||
}
|
||||
|
||||
if globalSiteReplicationSys.isEnabled() {
|
||||
configData := []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>speedtest/*</Prefix></ExcludedPrefixes></VersioningConfiguration>`)
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketVersioningConfig, configData); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
return bucketExists, nil
|
||||
}
|
||||
|
||||
@@ -1382,7 +1456,7 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
|
||||
func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1503,7 +1577,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.TraceAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -1518,7 +1592,9 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
|
||||
// Use buffered channel to take care of burst sends or slow w.Write()
|
||||
traceCh := make(chan madmin.TraceInfo, 4000)
|
||||
|
||||
// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
|
||||
traceCh := make(chan madmin.TraceInfo, 100000)
|
||||
|
||||
peers, _ := newPeerRestClients(globalEndpoints)
|
||||
|
||||
@@ -1542,7 +1618,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
peer.Trace(traceCh, ctx.Done(), traceOpts)
|
||||
}
|
||||
|
||||
keepAliveTicker := time.NewTicker(500 * time.Millisecond)
|
||||
keepAliveTicker := time.NewTicker(time.Second)
|
||||
defer keepAliveTicker.Stop()
|
||||
|
||||
enc := json.NewEncoder(w)
|
||||
@@ -1574,7 +1650,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
|
||||
func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConsoleLogAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1655,7 +1731,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
|
||||
func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSCreateKeyAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1676,7 +1752,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
|
||||
func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1713,7 +1789,7 @@ func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Reques
|
||||
func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -1837,7 +1913,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
|
||||
}
|
||||
}
|
||||
|
||||
log, audit := fetchLoggerInfo()
|
||||
log, audit := fetchLoggerInfo(ctx)
|
||||
|
||||
// Get the notification target info
|
||||
notifyTarget := fetchLambdaInfo()
|
||||
@@ -1917,7 +1993,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
|
||||
Domain: domain,
|
||||
Region: globalSite.Region,
|
||||
SQSARN: globalEventNotifier.GetARNList(false),
|
||||
DeploymentID: globalDeploymentID,
|
||||
DeploymentID: globalDeploymentID(),
|
||||
Buckets: buckets,
|
||||
Objects: objects,
|
||||
Versions: versions,
|
||||
@@ -1962,8 +2038,14 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
|
||||
|
||||
func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) {
|
||||
hostAnonymizer := createHostAnonymizer()
|
||||
// anonAddr - Anonymizes hosts in given input string.
|
||||
|
||||
anonParam := query.Get(anonymizeParam)
|
||||
// anonAddr - Anonymizes hosts in given input string
|
||||
// (only if the anonymize param is set to srict).
|
||||
anonAddr := func(addr string) string {
|
||||
if anonParam != anonymizeStrict {
|
||||
return addr
|
||||
}
|
||||
newAddr, found := hostAnonymizer[addr]
|
||||
if found {
|
||||
return newAddr
|
||||
@@ -2024,6 +2106,20 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteNetInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypeSysNet)) == "true" {
|
||||
localNetInfo := madmin.GetNetInfo(globalLocalNodeName, globalInternodeInterface)
|
||||
healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, localNetInfo)
|
||||
|
||||
peerNetInfos := globalNotificationSys.GetNetInfo(healthCtx)
|
||||
for _, n := range peerNetInfos {
|
||||
anonymizeAddr(&n)
|
||||
healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, n)
|
||||
}
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteOSInfo := func() {
|
||||
if query.Get("sysosinfo") == "true" {
|
||||
localOSInfo := madmin.GetOSInfo(healthCtx, globalLocalNodeName)
|
||||
@@ -2124,6 +2220,10 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
}
|
||||
|
||||
anonymizeCmdLine := func(cmdLine string) string {
|
||||
if anonParam != anonymizeStrict {
|
||||
return cmdLine
|
||||
}
|
||||
|
||||
if !globalIsDistErasure {
|
||||
// FS mode - single server - hard code to `server1`
|
||||
anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
|
||||
@@ -2245,6 +2345,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
getAndWritePlatformInfo()
|
||||
getAndWriteCPUs()
|
||||
getAndWritePartitions()
|
||||
getAndWriteNetInfo()
|
||||
getAndWriteOSInfo()
|
||||
getAndWriteMemInfo()
|
||||
getAndWriteProcInfo()
|
||||
@@ -2313,7 +2414,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
|
||||
ctx := r.Context()
|
||||
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
|
||||
objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
|
||||
if objectAPI == nil {
|
||||
return
|
||||
}
|
||||
@@ -2327,7 +2428,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
DeploymentID: globalDeploymentID(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -2402,11 +2503,27 @@ func getTLSInfo() madmin.TLSInfo {
|
||||
|
||||
if globalIsTLS {
|
||||
for _, c := range globalPublicCerts {
|
||||
check := xxh3.Hash(c.RawIssuer)
|
||||
check ^= xxh3.Hash(c.RawSubjectPublicKeyInfo)
|
||||
// We XOR, so order doesn't matter.
|
||||
for _, v := range c.DNSNames {
|
||||
check ^= xxh3.HashString(v)
|
||||
}
|
||||
for _, v := range c.EmailAddresses {
|
||||
check ^= xxh3.HashString(v)
|
||||
}
|
||||
for _, v := range c.IPAddresses {
|
||||
check ^= xxh3.HashString(v.String())
|
||||
}
|
||||
for _, v := range c.URIs {
|
||||
check ^= xxh3.HashString(v.String())
|
||||
}
|
||||
tlsInfo.Certs = append(tlsInfo.Certs, madmin.TLSCert{
|
||||
PubKeyAlgo: c.PublicKeyAlgorithm.String(),
|
||||
SignatureAlgo: c.SignatureAlgorithm.String(),
|
||||
NotBefore: c.NotBefore,
|
||||
NotAfter: c.NotAfter,
|
||||
Checksum: strconv.FormatUint(check, 16),
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -2420,7 +2537,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
ctx := r.Context()
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ServerInfoAdminAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -2457,7 +2574,7 @@ func assignPoolNumbers(servers []madmin.ServerProperties) {
|
||||
func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus {
|
||||
lambdaMap := make(map[string][]madmin.TargetIDStatus)
|
||||
|
||||
for _, tgt := range globalNotifyTargetList.Targets() {
|
||||
for _, tgt := range globalEventNotifier.Targets() {
|
||||
targetIDStatus := make(map[string]madmin.Status)
|
||||
active, _ := tgt.IsActive()
|
||||
targetID := tgt.ID()
|
||||
@@ -2545,31 +2662,38 @@ func fetchKMSStatusV2(ctx context.Context) []madmin.KMS {
|
||||
return stats
|
||||
}
|
||||
|
||||
func targetStatus(ctx context.Context, h logger.Target) madmin.Status {
|
||||
if h.IsOnline(ctx) {
|
||||
return madmin.Status{Status: string(madmin.ItemOnline)}
|
||||
}
|
||||
return madmin.Status{Status: string(madmin.ItemOffline)}
|
||||
}
|
||||
|
||||
// fetchLoggerDetails return log info
|
||||
func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
|
||||
func fetchLoggerInfo(ctx context.Context) ([]madmin.Logger, []madmin.Audit) {
|
||||
var loggerInfo []madmin.Logger
|
||||
var auditloggerInfo []madmin.Audit
|
||||
for _, tgt := range logger.SystemTargets() {
|
||||
if tgt.Endpoint() != "" {
|
||||
loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
|
||||
loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): targetStatus(ctx, tgt)})
|
||||
}
|
||||
}
|
||||
|
||||
for _, tgt := range logger.AuditTargets() {
|
||||
if tgt.Endpoint() != "" {
|
||||
auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
|
||||
auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): targetStatus(ctx, tgt)})
|
||||
}
|
||||
}
|
||||
|
||||
return loggerInfo, auditloggerInfo
|
||||
}
|
||||
|
||||
func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
|
||||
func embedFileInZip(zipWriter *zip.Writer, name string, data []byte, fileMode os.FileMode) error {
|
||||
// Send profiling data to zip as file
|
||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||
name: name,
|
||||
size: int64(len(data)),
|
||||
mode: 0o600,
|
||||
mode: fileMode,
|
||||
modTime: UTCNow(),
|
||||
isDir: false,
|
||||
sys: nil,
|
||||
@@ -2611,7 +2735,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
|
||||
ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
|
||||
ci.Info.MinioVersion = Version
|
||||
|
||||
si := objectAPI.StorageInfo(ctx)
|
||||
si := objectAPI.StorageInfo(ctx, true)
|
||||
|
||||
ci.Info.NoOfDrives = len(si.Disks)
|
||||
for _, disk := range si.Disks {
|
||||
@@ -2625,7 +2749,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
|
||||
ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount
|
||||
ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount
|
||||
|
||||
ci.DeploymentID = globalDeploymentID
|
||||
ci.DeploymentID = globalDeploymentID()
|
||||
ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
|
||||
|
||||
select {
|
||||
@@ -2672,7 +2796,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
ctx := r.Context()
|
||||
|
||||
// Validate request signature.
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.InspectDataAction, "")
|
||||
_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.InspectDataAction, "")
|
||||
if adminAPIErr != ErrNone {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
|
||||
return
|
||||
@@ -2800,7 +2924,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
defer inspectZipW.Close()
|
||||
|
||||
if b := getClusterMetaInfo(ctx); len(b) > 0 {
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b))
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2864,11 +2988,16 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
|
||||
sb.WriteString(pool.CmdLine)
|
||||
}
|
||||
sb.WriteString("\n")
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))
|
||||
|
||||
scheme := "https"
|
||||
if !globalIsTLS {
|
||||
scheme = "http"
|
||||
}
|
||||
|
||||
// save MinIO start script to inspect command
|
||||
var scrb bytes.Buffer
|
||||
scrb.WriteString(`#!/usr/bin/env bash
|
||||
fmt.Fprintf(&scrb, `#!/usr/bin/env bash
|
||||
|
||||
function main() {
|
||||
for file in $(ls -1); do
|
||||
@@ -2877,10 +3006,10 @@ function main() {
|
||||
done
|
||||
|
||||
# Read content of inspect-input.txt
|
||||
MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#https:\/\/#\.\/#g")
|
||||
MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#%s:\/\/#\.\/#g")
|
||||
|
||||
# Start MinIO instance using the options
|
||||
START_CMD="CI=on MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server ${MINIO_OPTS} &"
|
||||
START_CMD="CI=on _MINIO_AUTO_DRIVE_HEALING=off minio server ${MINIO_OPTS} &"
|
||||
echo
|
||||
echo "Starting MinIO instance: ${START_CMD}"
|
||||
echo
|
||||
@@ -2892,9 +3021,8 @@ function main() {
|
||||
sleep 10
|
||||
}
|
||||
|
||||
main "$@"`,
|
||||
)
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes()))
|
||||
main "$@"`, scheme)
|
||||
logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
|
||||
}
|
||||
|
||||
func getSubnetAdminPublicKey() []byte {
|
||||
|
||||
+5
-12
@@ -253,7 +253,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
|
||||
} else {
|
||||
clientToken := he.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s@%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
clientToken = fmt.Sprintf("%s:%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
|
||||
hsp = madmin.HealStopSuccess{
|
||||
@@ -327,7 +327,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
|
||||
|
||||
clientToken := h.clientToken
|
||||
if globalIsDistErasure {
|
||||
clientToken = fmt.Sprintf("%s@%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
|
||||
}
|
||||
|
||||
b, err := json.Marshal(madmin.HealStartSuccess{
|
||||
@@ -683,13 +683,6 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
|
||||
}
|
||||
}
|
||||
|
||||
func (h *healSequence) logHeal(healType madmin.HealItemType) {
|
||||
h.mutex.Lock()
|
||||
h.scannedItemsMap[healType]++
|
||||
h.lastHealActivity = UTCNow()
|
||||
h.mutex.Unlock()
|
||||
}
|
||||
|
||||
func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
|
||||
// Send heal request
|
||||
task := healTask{
|
||||
@@ -817,7 +810,7 @@ func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) f
|
||||
// NOTE: Healing on meta is run regardless
|
||||
// of any bucket being selected, this is to ensure that
|
||||
// meta are always upto date and correct.
|
||||
return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
|
||||
return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
@@ -874,7 +867,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
|
||||
|
||||
if !h.settings.Recursive {
|
||||
if h.object != "" {
|
||||
if err := h.healObject(bucket, h.object, ""); err != nil {
|
||||
if err := h.healObject(bucket, h.object, "", h.settings.ScanMode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -889,7 +882,7 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
|
||||
}
|
||||
|
||||
// healObject - heal the given object and record result
|
||||
func (h *healSequence) healObject(bucket, object, versionID string) error {
|
||||
func (h *healSequence) healObject(bucket, object, versionID string, scanMode madmin.HealScanMode) error {
|
||||
if h.isQuitting() {
|
||||
return errHealStopSignalled
|
||||
}
|
||||
|
||||
@@ -36,6 +36,8 @@ const (
|
||||
adminAPIVersionPrefix = SlashSeparator + adminAPIVersion
|
||||
adminAPISiteReplicationDevNull = "/site-replication/devnull"
|
||||
adminAPISiteReplicationNetPerf = "/site-replication/netperf"
|
||||
adminAPIClientDevNull = "/speedtest/client/devnull"
|
||||
adminAPIClientDevExtraTime = "/speedtest/client/devnull/extratime"
|
||||
)
|
||||
|
||||
var gzipHandler = func() func(http.Handler) http.HandlerFunc {
|
||||
@@ -298,6 +300,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.GetIdentityProviderCfg))
|
||||
adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.DeleteIdentityProviderCfg))
|
||||
|
||||
// LDAP specific service accounts ops
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp/ldap/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccountLDAP))
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys").
|
||||
HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).
|
||||
Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}")
|
||||
|
||||
// LDAP IAM operations
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP))
|
||||
@@ -374,6 +382,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(adminMiddleware(adminAPI.SRPeerEdit))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(adminMiddleware(adminAPI.SRPeerRemove))
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationResyncOp)).Queries("operation", "{operation:.*}")
|
||||
adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/state/edit").HandlerFunc(adminMiddleware(adminAPI.SRStateEdit))
|
||||
|
||||
if globalIsDistErasure {
|
||||
// Top locks
|
||||
@@ -388,6 +397,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(adminMiddleware(adminAPI.DriveSpeedtestHandler, noGZFlag))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(adminMiddleware(adminAPI.NetperfHandler, noGZFlag))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/site").HandlerFunc(adminMiddleware(adminAPI.SitePerfHandler, noGZFlag))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevNull).HandlerFunc(adminMiddleware(adminAPI.ClientDevNull, noGZFlag))
|
||||
adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevExtraTime).HandlerFunc(adminMiddleware(adminAPI.ClientDevNullExtraTime, noGZFlag))
|
||||
|
||||
// HTTP Trace
|
||||
adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminMiddleware(adminAPI.TraceHandler, noObjLayerFlag))
|
||||
|
||||
@@ -141,7 +141,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
|
||||
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer != nil {
|
||||
storageInfo := objLayer.LocalStorageInfo(GlobalContext)
|
||||
storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
|
||||
props.State = string(madmin.ItemOnline)
|
||||
props.Disks = storageInfo.Disks
|
||||
} else {
|
||||
|
||||
+21
-3
@@ -47,7 +47,7 @@ import (
|
||||
levent "github.com/minio/minio/internal/config/lambda/event"
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// APIError structure
|
||||
@@ -310,6 +310,7 @@ const (
|
||||
ErrSiteReplicationBucketMetaError
|
||||
ErrSiteReplicationIAMError
|
||||
ErrSiteReplicationConfigMissing
|
||||
ErrSiteReplicationIAMConfigMismatch
|
||||
|
||||
// Pool rebalance errors
|
||||
ErrAdminRebalanceAlreadyStarted
|
||||
@@ -429,6 +430,9 @@ const (
|
||||
ErrLambdaARNInvalid
|
||||
ErrLambdaARNNotFound
|
||||
|
||||
// New Codes for GetObjectAttributes and GetObjectVersionAttributes
|
||||
ErrInvalidAttributeName
|
||||
|
||||
apiErrCodeEnd // This is used only for the testing code
|
||||
)
|
||||
|
||||
@@ -1512,6 +1516,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Site not found in site replication configuration",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrSiteReplicationIAMConfigMismatch: {
|
||||
Code: "XMinioSiteReplicationIAMConfigMismatch",
|
||||
Description: "IAM configuration mismatch between sites",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminRebalanceAlreadyStarted: {
|
||||
Code: "XMinioAdminRebalanceAlreadyStarted",
|
||||
Description: "Pool rebalance is already started",
|
||||
@@ -2057,6 +2066,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "The specified policy is not found.",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
},
|
||||
ErrInvalidAttributeName: {
|
||||
Code: "InvalidArgument",
|
||||
Description: "Invalid attribute name specified.",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
// Add your error structure here.
|
||||
}
|
||||
|
||||
@@ -2236,8 +2250,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
apiErr = ErrSlowDownWrite
|
||||
case InsufficientReadQuorum:
|
||||
apiErr = ErrSlowDownRead
|
||||
case InvalidMarkerPrefixCombination:
|
||||
apiErr = ErrNotImplemented
|
||||
case InvalidUploadIDKeyCombination:
|
||||
apiErr = ErrNotImplemented
|
||||
case MalformedUploadID:
|
||||
@@ -2339,6 +2351,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
case dns.ErrBucketConflict:
|
||||
apiErr = ErrBucketAlreadyExists
|
||||
default:
|
||||
if _, ok := err.(tags.Error); ok {
|
||||
// tag errors are not exported, so we check their custom interface to avoid logging.
|
||||
// The correct type is inserted by toAPIError.
|
||||
apiErr = ErrInternalError
|
||||
break
|
||||
}
|
||||
var ie, iw int
|
||||
// This work-around is to handle the issue golang/go#30648
|
||||
//nolint:gocritic
|
||||
|
||||
@@ -42,7 +42,6 @@ var toAPIErrorTests = []struct {
|
||||
{err: InvalidPart{}, errCode: ErrInvalidPart},
|
||||
{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
|
||||
{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
|
||||
{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
|
||||
{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
|
||||
{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
|
||||
{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
|
||||
|
||||
@@ -133,11 +133,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
||||
w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
|
||||
}
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
|
||||
w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
|
||||
}
|
||||
|
||||
// Set tag count if object has tags
|
||||
if len(objInfo.UserTags) > 0 {
|
||||
tags, _ := tags.ParseObjectTags(objInfo.UserTags)
|
||||
|
||||
+48
-22
@@ -35,7 +35,7 @@ import (
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
xxml "github.com/minio/xxml"
|
||||
)
|
||||
|
||||
@@ -502,6 +502,47 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
|
||||
return data
|
||||
}
|
||||
|
||||
func cleanReservedKeys(metadata map[string]string) map[string]string {
|
||||
m := cloneMSS(metadata)
|
||||
|
||||
switch kind, _ := crypto.IsEncrypted(metadata); kind {
|
||||
case crypto.S3:
|
||||
m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES
|
||||
case crypto.S3KMS:
|
||||
m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS
|
||||
m[xhttp.AmzServerSideEncryptionKmsID] = kmsKeyIDFromMetadata(metadata)
|
||||
if kmsCtx, ok := metadata[crypto.MetaContext]; ok {
|
||||
m[xhttp.AmzServerSideEncryptionKmsContext] = kmsCtx
|
||||
}
|
||||
case crypto.SSEC:
|
||||
m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES
|
||||
|
||||
}
|
||||
|
||||
var toRemove []string
|
||||
for k := range cleanMinioInternalMetadataKeys(m) {
|
||||
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||
// Do not need to send any internal metadata
|
||||
// values to client.
|
||||
toRemove = append(toRemove, k)
|
||||
continue
|
||||
}
|
||||
|
||||
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
||||
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
||||
toRemove = append(toRemove, k)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
for _, k := range toRemove {
|
||||
delete(m, k)
|
||||
delete(m, strings.ToLower(k))
|
||||
}
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
// generates an ListBucketVersions response for the said bucket with other enumerated options.
|
||||
func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
|
||||
versions := make([]ObjectVersion, 0, len(resp.Objects))
|
||||
@@ -549,18 +590,11 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
||||
case crypto.SSEC:
|
||||
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
|
||||
}
|
||||
for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
|
||||
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||
// Do not need to send any internal metadata
|
||||
// values to client.
|
||||
continue
|
||||
}
|
||||
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
||||
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
||||
continue
|
||||
}
|
||||
for k, v := range cleanReservedKeys(object.UserDefined) {
|
||||
content.UserMetadata.Set(k, v)
|
||||
}
|
||||
|
||||
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||
content.Internal = &ObjectInternalInfo{
|
||||
K: object.DataBlocks,
|
||||
M: object.ParityBlocks,
|
||||
@@ -692,18 +726,10 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
||||
case crypto.SSEC:
|
||||
content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
|
||||
}
|
||||
for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
|
||||
if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
|
||||
// Do not need to send any internal metadata
|
||||
// values to client.
|
||||
continue
|
||||
}
|
||||
// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
|
||||
if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
|
||||
continue
|
||||
}
|
||||
for k, v := range cleanReservedKeys(object.UserDefined) {
|
||||
content.UserMetadata.Set(k, v)
|
||||
}
|
||||
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||
content.Internal = &ObjectInternalInfo{
|
||||
K: object.DataBlocks,
|
||||
M: object.ParityBlocks,
|
||||
@@ -978,7 +1004,7 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
|
||||
BucketName: reqInfo.BucketName,
|
||||
Key: reqInfo.ObjectName,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
HostID: globalDeploymentID(),
|
||||
}
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||
|
||||
+11
-19
@@ -23,11 +23,11 @@ import (
|
||||
"net/http"
|
||||
|
||||
"github.com/klauspost/compress/gzhttp"
|
||||
"github.com/minio/console/restapi"
|
||||
consoleapi "github.com/minio/console/api"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/wildcard"
|
||||
"github.com/minio/pkg/v2/wildcard"
|
||||
"github.com/rs/cors"
|
||||
)
|
||||
|
||||
@@ -43,13 +43,13 @@ func setHTTPServer(h *xhttp.Server) {
|
||||
globalObjLayerMutex.Unlock()
|
||||
}
|
||||
|
||||
func newConsoleServerFn() *restapi.Server {
|
||||
func newConsoleServerFn() *consoleapi.Server {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
return globalConsoleSrv
|
||||
}
|
||||
|
||||
func setConsoleSrv(srv *restapi.Server) {
|
||||
func setConsoleSrv(srv *consoleapi.Server) {
|
||||
globalObjLayerMutex.Lock()
|
||||
globalConsoleSrv = srv
|
||||
globalObjLayerMutex.Unlock()
|
||||
@@ -61,18 +61,6 @@ func newObjectLayerFn() ObjectLayer {
|
||||
return globalObjectAPI
|
||||
}
|
||||
|
||||
func newCachedObjectLayerFn() CacheObjectLayer {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
return globalCacheObjectAPI
|
||||
}
|
||||
|
||||
func setCacheObjectLayer(c CacheObjectLayer) {
|
||||
globalObjLayerMutex.Lock()
|
||||
globalCacheObjectAPI = c
|
||||
globalObjLayerMutex.Unlock()
|
||||
}
|
||||
|
||||
func setObjectLayer(o ObjectLayer) {
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = o
|
||||
@@ -82,7 +70,6 @@ func setObjectLayer(o ObjectLayer) {
|
||||
// objectAPIHandler implements and provides http handlers for S3 API.
|
||||
type objectAPIHandlers struct {
|
||||
ObjectAPI func() ObjectLayer
|
||||
CacheAPI func() CacheObjectLayer
|
||||
}
|
||||
|
||||
// getHost tries its best to return the request host.
|
||||
@@ -189,7 +176,6 @@ func registerAPIRouter(router *mux.Router) {
|
||||
// Initialize API.
|
||||
api := objectAPIHandlers{
|
||||
ObjectAPI: newObjectLayerFn,
|
||||
CacheAPI: newCachedObjectLayerFn,
|
||||
}
|
||||
|
||||
// API Router
|
||||
@@ -241,6 +227,9 @@ func registerAPIRouter(router *mux.Router) {
|
||||
// HeadObject
|
||||
router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
|
||||
collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
|
||||
// GetObjectAttribytes
|
||||
router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
|
||||
collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "")
|
||||
// CopyObjectPart
|
||||
router.Methods(http.MethodPut).Path("/{object:.+}").
|
||||
HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
|
||||
@@ -461,9 +450,12 @@ func registerAPIRouter(router *mux.Router) {
|
||||
|
||||
// MinIO extension API for replication.
|
||||
//
|
||||
// GetBucketReplicationMetrics
|
||||
router.Methods(http.MethodGet).HandlerFunc(
|
||||
collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2")
|
||||
// deprecated handler
|
||||
router.Methods(http.MethodGet).HandlerFunc(
|
||||
collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
|
||||
|
||||
// ValidateBucketReplicationCreds
|
||||
router.Methods(http.MethodGet).HandlerFunc(
|
||||
collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
|
||||
|
||||
+112
-110
File diff suppressed because one or more lines are too long
+23
-24
@@ -41,8 +41,7 @@ import (
|
||||
xjwt "github.com/minio/minio/internal/jwt"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/mcontext"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// Verify if request has JWT.
|
||||
@@ -186,15 +185,15 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
|
||||
// checkAdminRequestAuth checks for authentication and authorization for the incoming
|
||||
// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
|
||||
// are automatically rejected.
|
||||
func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
|
||||
func checkAdminRequestAuth(ctx context.Context, r *http.Request, action policy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
|
||||
cred, owner, s3Err := validateAdminSignature(ctx, r, region)
|
||||
if s3Err != ErrNone {
|
||||
return cred, s3Err
|
||||
}
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.Action(action),
|
||||
Action: policy.Action(action),
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
Claims: cred.Claims,
|
||||
@@ -248,7 +247,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
|
||||
}
|
||||
|
||||
// Check if a session policy is set. If so, decode it here.
|
||||
sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
|
||||
sp, spok := claims.Lookup(policy.SessionPolicyName)
|
||||
if spok {
|
||||
// Looks like subpolicy is set and is a string, if set then its
|
||||
// base64 encoded, decode it. Decoding fails reject such
|
||||
@@ -413,7 +412,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
|
||||
if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
|
||||
// Anonymous checks are not meant for ListAllBuckets action
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
@@ -429,7 +428,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
if action == policy.ListBucketVersionsAction {
|
||||
// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
|
||||
// verify as a fallback.
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.ListBucketAction,
|
||||
@@ -446,10 +445,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
return ErrAccessDenied
|
||||
}
|
||||
if action == policy.DeleteObjectAction && versionID != "" {
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.Action(policy.DeleteObjectVersionAction),
|
||||
Action: policy.Action(policy.DeleteObjectVersionAction),
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
ObjectName: object,
|
||||
@@ -460,10 +459,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
return ErrAccessDenied
|
||||
}
|
||||
}
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.Action(action),
|
||||
Action: action,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
ObjectName: object,
|
||||
@@ -477,10 +476,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
|
||||
if action == policy.ListBucketVersionsAction {
|
||||
// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
|
||||
// verify as a fallback.
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListBucketAction,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
ObjectName: object,
|
||||
@@ -568,7 +567,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
|
||||
|
||||
// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
|
||||
// The verification happens implicit during reading.
|
||||
reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
|
||||
reader, err := hash.NewReader(ctx, r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
|
||||
if err != nil {
|
||||
return toAPIErrorCode(ctx, err)
|
||||
}
|
||||
@@ -696,10 +695,10 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)}
|
||||
}
|
||||
if retMode == objectlock.RetGovernance && byPassSet {
|
||||
byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
byPassSet = globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.BypassGovernanceRetentionAction,
|
||||
Action: policy.BypassGovernanceRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ObjectName: objectName,
|
||||
ConditionValues: conditions,
|
||||
@@ -707,10 +706,10 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
Claims: cred.Claims,
|
||||
})
|
||||
}
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.PutObjectRetentionAction,
|
||||
Action: policy.PutObjectRetentionAction,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: conditions,
|
||||
ObjectName: objectName,
|
||||
@@ -728,7 +727,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
|
||||
// isPutActionAllowed - check if PUT operation is allowed on the resource, this
|
||||
// call verifies bucket policies and IAM policies, supports multi user
|
||||
// checks etc.
|
||||
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
|
||||
func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
|
||||
var cred auth.Credentials
|
||||
var owner bool
|
||||
region := globalSite.Region
|
||||
@@ -751,17 +750,17 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
||||
// Do not check for PutObjectRetentionAction permission,
|
||||
// if mode and retain until date are not set.
|
||||
// Can happen when bucket has default lock config set
|
||||
if action == iampolicy.PutObjectRetentionAction &&
|
||||
if action == policy.PutObjectRetentionAction &&
|
||||
r.Header.Get(xhttp.AmzObjectLockMode) == "" &&
|
||||
r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" {
|
||||
return ErrNone
|
||||
}
|
||||
|
||||
if cred.AccessKey == "" {
|
||||
if globalPolicySys.IsAllowed(policy.Args{
|
||||
if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: policy.Action(action),
|
||||
Action: action,
|
||||
BucketName: bucketName,
|
||||
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
||||
IsOwner: false,
|
||||
@@ -772,7 +771,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
|
||||
return ErrAccessDenied
|
||||
}
|
||||
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
type nullReader struct{}
|
||||
@@ -443,7 +443,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
|
||||
{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
|
||||
}
|
||||
for i, testCase := range testCases {
|
||||
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
|
||||
if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
|
||||
t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/env"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
)
|
||||
|
||||
// healTask represents what to heal along with options
|
||||
|
||||
@@ -34,7 +34,7 @@ import (
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/env"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -232,7 +232,7 @@ func (h *healingTracker) delete(ctx context.Context) error {
|
||||
pathJoin(bucketMetaPrefix, healingTrackerFilename),
|
||||
DeleteOptions{
|
||||
Recursive: false,
|
||||
Force: false,
|
||||
Immediate: false,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -345,18 +345,17 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
|
||||
|
||||
initBackgroundHealing(ctx, objAPI) // start quick background healing
|
||||
|
||||
globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
|
||||
|
||||
if env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
|
||||
if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
|
||||
globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
|
||||
go monitorLocalDisksAndHeal(ctx, z)
|
||||
}
|
||||
}
|
||||
|
||||
func getLocalDisksToHeal() (disksToHeal Endpoints) {
|
||||
globalLocalDrivesMu.RLock()
|
||||
globalLocalDrives := globalLocalDrives
|
||||
localDrives := globalLocalDrives
|
||||
globalLocalDrivesMu.RUnlock()
|
||||
for _, disk := range globalLocalDrives {
|
||||
for _, disk := range localDrives {
|
||||
_, err := disk.GetDiskID()
|
||||
if errors.Is(err, errUnformattedDisk) {
|
||||
disksToHeal = append(disksToHeal, disk.Endpoint())
|
||||
|
||||
@@ -0,0 +1,677 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/tags"
|
||||
"github.com/minio/minio/internal/bucket/versioning"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
"github.com/minio/pkg/v2/wildcard"
|
||||
"github.com/minio/pkg/v2/workers"
|
||||
)
|
||||
|
||||
// expire: # Expire objects that match a condition
|
||||
// apiVersion: v1
|
||||
// bucket: mybucket # Bucket where this batch job will expire matching objects from
|
||||
// prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
|
||||
// rules:
|
||||
// - type: object # regular objects with zero ore more older versions
|
||||
// name: NAME # match object names that satisfy the wildcard expression.
|
||||
// olderThan: 70h # match objects older than this value
|
||||
// createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||
// tags:
|
||||
// - key: name
|
||||
// value: pick* # match objects with tag 'name', all values starting with 'pick'
|
||||
// metadata:
|
||||
// - key: content-type
|
||||
// value: image/* # match objects with 'content-type', all values starting with 'image/'
|
||||
// size:
|
||||
// lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
|
||||
// greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
|
||||
// purge:
|
||||
// # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||
// # retainVersions: 5 # keep the latest 5 versions of the object.
|
||||
//
|
||||
// - type: deleted # objects with delete marker as their latest version
|
||||
// name: NAME # match object names that satisfy the wildcard expression.
|
||||
// olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
|
||||
// createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||
// purge:
|
||||
// # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||
// # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
|
||||
//
|
||||
// notify:
|
||||
// endpoint: https://notify.endpoint # notification endpoint to receive job completion status
|
||||
// token: Bearer xxxxx # optional authentication token for the notification endpoint
|
||||
//
|
||||
// retry:
|
||||
// attempts: 10 # number of retries for the job before giving up
|
||||
// delay: 500ms # least amount of delay between each retry
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
// BatchJobExpirePurge type accepts non-negative versions to be retained
|
||||
type BatchJobExpirePurge struct {
|
||||
RetainVersions int `yaml:"retainVersions" json:"retainVersions"`
|
||||
}
|
||||
|
||||
// Validate returns nil if value is valid, ie > 0.
|
||||
func (p BatchJobExpirePurge) Validate() error {
|
||||
if p.RetainVersions < 0 {
|
||||
return errors.New("retainVersions must be >= 0")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchJobExpireFilter holds all the filters currently supported for batch replication
|
||||
type BatchJobExpireFilter struct {
|
||||
OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
|
||||
CreatedBefore *time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"`
|
||||
Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"`
|
||||
Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"`
|
||||
Size BatchJobSizeFilter `yaml:"size" json:"size"`
|
||||
Type string `yaml:"type" json:"type"`
|
||||
Name string `yaml:"name" json:"name"`
|
||||
Purge BatchJobExpirePurge `yaml:"purge" json:"purge"`
|
||||
}
|
||||
|
||||
// Matches returns true if obj matches the filter conditions specified in ef.
|
||||
func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
|
||||
switch ef.Type {
|
||||
case BatchJobExpireObject:
|
||||
if obj.DeleteMarker {
|
||||
return false
|
||||
}
|
||||
case BatchJobExpireDeleted:
|
||||
if !obj.DeleteMarker {
|
||||
return false
|
||||
}
|
||||
default:
|
||||
// we should never come here, Validate should have caught this.
|
||||
logger.LogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type)
|
||||
return false
|
||||
}
|
||||
|
||||
if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) {
|
||||
return false
|
||||
}
|
||||
if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan {
|
||||
return false
|
||||
}
|
||||
|
||||
if ef.CreatedBefore != nil && !obj.ModTime.Before(*ef.CreatedBefore) {
|
||||
return false
|
||||
}
|
||||
|
||||
if len(ef.Tags) > 0 && !obj.DeleteMarker {
|
||||
// Only parse object tags if tags filter is specified.
|
||||
var tagMap map[string]string
|
||||
if len(obj.UserTags) != 0 {
|
||||
t, err := tags.ParseObjectTags(obj.UserTags)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
tagMap = t.ToMap()
|
||||
}
|
||||
|
||||
for _, kv := range ef.Tags {
|
||||
// Object (version) must match all tags specified in
|
||||
// the filter
|
||||
var match bool
|
||||
for t, v := range tagMap {
|
||||
if kv.Match(BatchJobKV{Key: t, Value: v}) {
|
||||
match = true
|
||||
}
|
||||
}
|
||||
if !match {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
if len(ef.Metadata) > 0 && !obj.DeleteMarker {
|
||||
for _, kv := range ef.Metadata {
|
||||
// Object (version) must match all x-amz-meta and
|
||||
// standard metadata headers
|
||||
// specified in the filter
|
||||
var match bool
|
||||
for k, v := range obj.UserDefined {
|
||||
if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
|
||||
continue
|
||||
}
|
||||
// We only need to match x-amz-meta or standardHeaders
|
||||
if kv.Match(BatchJobKV{Key: k, Value: v}) {
|
||||
match = true
|
||||
}
|
||||
}
|
||||
if !match {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return ef.Size.InRange(obj.Size)
|
||||
}
|
||||
|
||||
const (
|
||||
// BatchJobExpireObject - object type
|
||||
BatchJobExpireObject string = "object"
|
||||
// BatchJobExpireDeleted - delete marker type
|
||||
BatchJobExpireDeleted string = "deleted"
|
||||
)
|
||||
|
||||
// Validate returns nil if ef has valid fields, validation error otherwise.
|
||||
func (ef BatchJobExpireFilter) Validate() error {
|
||||
switch ef.Type {
|
||||
case BatchJobExpireObject:
|
||||
case BatchJobExpireDeleted:
|
||||
if len(ef.Tags) > 0 || len(ef.Metadata) > 0 {
|
||||
return errors.New("invalid batch-expire rule filter")
|
||||
}
|
||||
default:
|
||||
return errors.New("invalid batch-expire type")
|
||||
}
|
||||
|
||||
for _, tag := range ef.Tags {
|
||||
if err := tag.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, meta := range ef.Metadata {
|
||||
if err := meta.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := ef.Purge.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := ef.Size.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
if ef.CreatedBefore != nil && !ef.CreatedBefore.Before(time.Now()) {
|
||||
return errors.New("CreatedBefore is in the future")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchJobExpire represents configuration parameters for a batch expiration
|
||||
// job typically supplied in yaml form
|
||||
type BatchJobExpire struct {
|
||||
APIVersion string `yaml:"apiVersion" json:"apiVersion"`
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
Prefix string `yaml:"prefix" json:"prefix"`
|
||||
NotificationCfg BatchJobNotification `yaml:"notify" json:"notify"`
|
||||
Retry BatchJobRetry `yaml:"retry" json:"retry"`
|
||||
Rules []BatchJobExpireFilter `yaml:"rules" json:"rules"`
|
||||
}
|
||||
|
||||
// Notify notifies notification endpoint if configured regarding job failure or success.
|
||||
func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error {
|
||||
if r.NotificationCfg.Endpoint == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.NotificationCfg.Endpoint, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if r.NotificationCfg.Token != "" {
|
||||
req.Header.Set("Authorization", r.NotificationCfg.Token)
|
||||
}
|
||||
|
||||
clnt := http.Client{Transport: getRemoteInstanceTransport}
|
||||
resp, err := clnt.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
xhttp.DrainBody(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errors.New(resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Expire expires object versions which have already matched supplied filter conditions
|
||||
func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error {
|
||||
opts := ObjectOptions{
|
||||
PrefixEnabledFn: vc.PrefixEnabled,
|
||||
VersionSuspended: vc.Suspended(),
|
||||
}
|
||||
_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel, opts)
|
||||
return errs
|
||||
}
|
||||
|
||||
const (
|
||||
batchExpireName = "batch-expire.bin"
|
||||
batchExpireFormat = 1
|
||||
batchExpireVersionV1 = 1
|
||||
batchExpireVersion = batchExpireVersionV1
|
||||
batchExpireAPIVersion = "v1"
|
||||
batchExpireJobDefaultRetries = 3
|
||||
batchExpireJobDefaultRetryDelay = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
type objInfoCache map[string]*ObjectInfo
|
||||
|
||||
func newObjInfoCache() objInfoCache {
|
||||
return objInfoCache(make(map[string]*ObjectInfo))
|
||||
}
|
||||
|
||||
func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) {
|
||||
oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi
|
||||
}
|
||||
|
||||
func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) {
|
||||
oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)]
|
||||
return oi, ok
|
||||
}
|
||||
|
||||
func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) {
|
||||
vc, _ := globalBucketVersioningSys.Get(r.Bucket)
|
||||
retryAttempts := r.Retry.Attempts
|
||||
delay := job.Expire.Retry.Delay
|
||||
if delay == 0 {
|
||||
delay = batchExpireJobDefaultRetryDelay
|
||||
}
|
||||
|
||||
var i int
|
||||
for toExpire := range expireCh {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
if i > 0 {
|
||||
if wait := globalBatchConfig.ExpirationWait(); wait > 0 {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}
|
||||
i++
|
||||
wk.Take()
|
||||
go func(toExpire []expireObjInfo) {
|
||||
defer wk.Give()
|
||||
|
||||
toExpireAll := make([]ObjectInfo, 0, len(toExpire))
|
||||
toDel := make([]ObjectToDelete, 0, len(toExpire))
|
||||
oiCache := newObjInfoCache()
|
||||
for _, exp := range toExpire {
|
||||
if exp.ExpireAll {
|
||||
toExpireAll = append(toExpireAll, exp.ObjectInfo)
|
||||
continue
|
||||
}
|
||||
// Cache ObjectInfo value via pointers for
|
||||
// subsequent use to track objects which
|
||||
// couldn't be deleted.
|
||||
od := ObjectToDelete{
|
||||
ObjectV: ObjectV{
|
||||
ObjectName: exp.Name,
|
||||
VersionID: exp.VersionID,
|
||||
},
|
||||
}
|
||||
toDel = append(toDel, od)
|
||||
oiCache.Add(od, &exp.ObjectInfo)
|
||||
}
|
||||
|
||||
var done bool
|
||||
// DeleteObject(deletePrefix: true) to expire all versions of an object
|
||||
for _, exp := range toExpireAll {
|
||||
var success bool
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
default:
|
||||
}
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
_, err := api.DeleteObject(ctx, exp.Bucket, exp.Name, ObjectOptions{
|
||||
DeletePrefix: true,
|
||||
})
|
||||
if err != nil {
|
||||
stopFn(exp, err)
|
||||
logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", toExpire[i].Bucket, toExpire[i].Name, toExpire[i].VersionID, err, attempts))
|
||||
} else {
|
||||
stopFn(exp, err)
|
||||
success = true
|
||||
break
|
||||
}
|
||||
}
|
||||
ri.trackMultipleObjectVersions(r.Bucket, exp, success)
|
||||
if done {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if done {
|
||||
return
|
||||
}
|
||||
|
||||
// DeleteMultiple objects
|
||||
toDelCopy := make([]ObjectToDelete, len(toDel))
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts)
|
||||
// Copying toDel to select from objects whose
|
||||
// deletion failed
|
||||
copy(toDelCopy, toDel)
|
||||
var failed int
|
||||
errs := r.Expire(ctx, api, vc, toDel)
|
||||
// reslice toDel in preparation for next retry
|
||||
// attempt
|
||||
toDel = toDel[:0]
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
stopFn(toDelCopy[i], err)
|
||||
logger.LogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, err, attempts))
|
||||
failed++
|
||||
if attempts == retryAttempts { // all retry attempts failed, record failure
|
||||
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
|
||||
ri.trackCurrentBucketObject(r.Bucket, *oi, false)
|
||||
}
|
||||
} else {
|
||||
toDel = append(toDel, toDelCopy[i])
|
||||
}
|
||||
} else {
|
||||
stopFn(toDelCopy[i], nil)
|
||||
if oi, ok := oiCache.Get(toDelCopy[i]); ok {
|
||||
ri.trackCurrentBucketObject(r.Bucket, *oi, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
globalBatchJobsMetrics.save(ri.JobID, ri)
|
||||
|
||||
if failed == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Add a delay between retry attempts
|
||||
if attempts < retryAttempts {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
}
|
||||
}(toExpire)
|
||||
}
|
||||
}
|
||||
|
||||
type expireObjInfo struct {
|
||||
ObjectInfo
|
||||
ExpireAll bool
|
||||
}
|
||||
|
||||
// Start the batch expiration job, resumes if there was a pending job via "job.ID"
|
||||
func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
|
||||
ri := &batchJobInfo{
|
||||
JobID: job.ID,
|
||||
JobType: string(job.Type()),
|
||||
StartTime: job.Started,
|
||||
}
|
||||
if err := ri.load(ctx, api, job); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
lastObject := ri.Object
|
||||
|
||||
now := time.Now().UTC()
|
||||
|
||||
workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_EXPIRATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
wk, err := workers.New(workerSize)
|
||||
if err != nil {
|
||||
// invalid worker size.
|
||||
return err
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
|
||||
results := make(chan ObjectInfo, workerSize)
|
||||
if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
|
||||
Marker: lastObject,
|
||||
LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions
|
||||
VersionsSort: WalkVersionsSortDesc,
|
||||
}); err != nil {
|
||||
// Do not need to retry if we can't list objects on source.
|
||||
return err
|
||||
}
|
||||
|
||||
// Goroutine to periodically save batch-expire job's in-memory state
|
||||
saverQuitCh := make(chan struct{})
|
||||
go func() {
|
||||
saveTicker := time.NewTicker(10 * time.Second)
|
||||
defer saveTicker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-saveTicker.C:
|
||||
// persist in-memory state to disk after every 10secs.
|
||||
logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
|
||||
|
||||
case <-ctx.Done():
|
||||
// persist in-memory state immediately before exiting due to context cancelation.
|
||||
logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
|
||||
return
|
||||
|
||||
case <-saverQuitCh:
|
||||
// persist in-memory state immediately to disk.
|
||||
logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
expireCh := make(chan []expireObjInfo, workerSize)
|
||||
go batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh)
|
||||
|
||||
var (
|
||||
prevObj ObjectInfo
|
||||
matchedFilter BatchJobExpireFilter
|
||||
versionsCount int
|
||||
toDel []expireObjInfo
|
||||
)
|
||||
for result := range results {
|
||||
// Apply filter to find the matching rule to apply expiry
|
||||
// actions accordingly.
|
||||
// nolint:gocritic
|
||||
if result.IsLatest {
|
||||
// send down filtered entries to be deleted using
|
||||
// DeleteObjects method
|
||||
if len(toDel) > 10 { // batch up to 10 objects/versions to be expired simultaneously.
|
||||
xfer := make([]expireObjInfo, len(toDel))
|
||||
copy(xfer, toDel)
|
||||
|
||||
var done bool
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
done = true
|
||||
case expireCh <- xfer:
|
||||
toDel = toDel[:0] // resetting toDel
|
||||
}
|
||||
if done {
|
||||
break
|
||||
}
|
||||
}
|
||||
var match BatchJobExpireFilter
|
||||
var found bool
|
||||
for _, rule := range r.Rules {
|
||||
if rule.Matches(result, now) {
|
||||
match = rule
|
||||
found = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
|
||||
prevObj = result
|
||||
matchedFilter = match
|
||||
versionsCount = 1
|
||||
// Include the latest version
|
||||
if matchedFilter.Purge.RetainVersions == 0 {
|
||||
toDel = append(toDel, expireObjInfo{
|
||||
ObjectInfo: result,
|
||||
ExpireAll: true,
|
||||
})
|
||||
continue
|
||||
}
|
||||
} else if prevObj.Name == result.Name {
|
||||
if matchedFilter.Purge.RetainVersions == 0 {
|
||||
continue // including latest version in toDel suffices, skipping other versions
|
||||
}
|
||||
versionsCount++
|
||||
} else {
|
||||
continue
|
||||
}
|
||||
|
||||
if versionsCount <= matchedFilter.Purge.RetainVersions {
|
||||
continue // retain versions
|
||||
}
|
||||
toDel = append(toDel, expireObjInfo{
|
||||
ObjectInfo: result,
|
||||
})
|
||||
}
|
||||
// Send any remaining objects downstream
|
||||
if len(toDel) > 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case expireCh <- toDel:
|
||||
}
|
||||
}
|
||||
close(expireCh)
|
||||
|
||||
wk.Wait() // waits for all expire goroutines to complete
|
||||
|
||||
ri.Complete = ri.ObjectsFailed == 0
|
||||
ri.Failed = ri.ObjectsFailed > 0
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
|
||||
// Close the saverQuitCh - this also triggers saving in-memory state
|
||||
// immediately one last time before we exit this method.
|
||||
close(saverQuitCh)
|
||||
|
||||
// Notify expire jobs final status to the configured endpoint
|
||||
buf, _ := json.Marshal(ri)
|
||||
if err := r.Notify(context.Background(), bytes.NewReader(buf)); err != nil {
|
||||
logger.LogIf(context.Background(), fmt.Errorf("unable to notify %v", err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
//msgp:ignore batchExpireJobError
|
||||
type batchExpireJobError struct {
|
||||
Code string
|
||||
Description string
|
||||
HTTPStatusCode int
|
||||
}
|
||||
|
||||
func (e batchExpireJobError) Error() string {
|
||||
return e.Description
|
||||
}
|
||||
|
||||
// maxBatchRules maximum number of rules a batch-expiry job supports
|
||||
const maxBatchRules = 50
|
||||
|
||||
// Validate validates the job definition input
|
||||
func (r *BatchJobExpire) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
|
||||
if r == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
if r.APIVersion != batchExpireAPIVersion {
|
||||
return batchExpireJobError{
|
||||
Code: "InvalidArgument",
|
||||
Description: "Unsupported batch expire API version",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
|
||||
if r.Bucket == "" {
|
||||
return batchExpireJobError{
|
||||
Code: "InvalidArgument",
|
||||
Description: "Bucket argument missing",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
|
||||
if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
|
||||
if isErrBucketNotFound(err) {
|
||||
return batchExpireJobError{
|
||||
Code: "NoSuchSourceBucket",
|
||||
Description: "The specified source bucket does not exist",
|
||||
HTTPStatusCode: http.StatusNotFound,
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
if len(r.Rules) > maxBatchRules {
|
||||
return batchExpireJobError{
|
||||
Code: "InvalidArgument",
|
||||
Description: "Too many rules. Batch expire job can't have more than 100 rules",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
|
||||
for _, rule := range r.Rules {
|
||||
if err := rule.Validate(); err != nil {
|
||||
return batchExpireJobError{
|
||||
Code: "InvalidArgument",
|
||||
Description: fmt.Sprintf("Invalid batch expire rule: %s", err),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if err := r.Retry.Validate(); err != nil {
|
||||
return batchExpireJobError{
|
||||
Code: "InvalidArgument",
|
||||
Description: fmt.Sprintf("Invalid batch expire retry configuration: %s", err),
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,856 @@
|
||||
package cmd
|
||||
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *BatchJobExpire) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "APIVersion":
|
||||
z.APIVersion, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "APIVersion")
|
||||
return
|
||||
}
|
||||
case "Bucket":
|
||||
z.Bucket, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "Prefix":
|
||||
z.Prefix, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
case "NotificationCfg":
|
||||
err = z.NotificationCfg.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "NotificationCfg")
|
||||
return
|
||||
}
|
||||
case "Retry":
|
||||
err = z.Retry.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Retry")
|
||||
return
|
||||
}
|
||||
case "Rules":
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Rules")
|
||||
return
|
||||
}
|
||||
if cap(z.Rules) >= int(zb0002) {
|
||||
z.Rules = (z.Rules)[:zb0002]
|
||||
} else {
|
||||
z.Rules = make([]BatchJobExpireFilter, zb0002)
|
||||
}
|
||||
for za0001 := range z.Rules {
|
||||
err = z.Rules[za0001].DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Rules", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BatchJobExpire) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 6
|
||||
// write "APIVersion"
|
||||
err = en.Append(0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.APIVersion)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "APIVersion")
|
||||
return
|
||||
}
|
||||
// write "Bucket"
|
||||
err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Bucket)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
// write "Prefix"
|
||||
err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Prefix)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
// write "NotificationCfg"
|
||||
err = en.Append(0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = z.NotificationCfg.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "NotificationCfg")
|
||||
return
|
||||
}
|
||||
// write "Retry"
|
||||
err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = z.Retry.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Retry")
|
||||
return
|
||||
}
|
||||
// write "Rules"
|
||||
err = en.Append(0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(len(z.Rules)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Rules")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.Rules {
|
||||
err = z.Rules[za0001].EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Rules", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BatchJobExpire) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 6
|
||||
// string "APIVersion"
|
||||
o = append(o, 0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e)
|
||||
o = msgp.AppendString(o, z.APIVersion)
|
||||
// string "Bucket"
|
||||
o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
|
||||
o = msgp.AppendString(o, z.Bucket)
|
||||
// string "Prefix"
|
||||
o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
|
||||
o = msgp.AppendString(o, z.Prefix)
|
||||
// string "NotificationCfg"
|
||||
o = append(o, 0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
|
||||
o, err = z.NotificationCfg.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "NotificationCfg")
|
||||
return
|
||||
}
|
||||
// string "Retry"
|
||||
o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
|
||||
o, err = z.Retry.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Retry")
|
||||
return
|
||||
}
|
||||
// string "Rules"
|
||||
o = append(o, 0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73)
|
||||
o = msgp.AppendArrayHeader(o, uint32(len(z.Rules)))
|
||||
for za0001 := range z.Rules {
|
||||
o, err = z.Rules[za0001].MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Rules", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *BatchJobExpire) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "APIVersion":
|
||||
z.APIVersion, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "APIVersion")
|
||||
return
|
||||
}
|
||||
case "Bucket":
|
||||
z.Bucket, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Bucket")
|
||||
return
|
||||
}
|
||||
case "Prefix":
|
||||
z.Prefix, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Prefix")
|
||||
return
|
||||
}
|
||||
case "NotificationCfg":
|
||||
bts, err = z.NotificationCfg.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "NotificationCfg")
|
||||
return
|
||||
}
|
||||
case "Retry":
|
||||
bts, err = z.Retry.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Retry")
|
||||
return
|
||||
}
|
||||
case "Rules":
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Rules")
|
||||
return
|
||||
}
|
||||
if cap(z.Rules) >= int(zb0002) {
|
||||
z.Rules = (z.Rules)[:zb0002]
|
||||
} else {
|
||||
z.Rules = make([]BatchJobExpireFilter, zb0002)
|
||||
}
|
||||
for za0001 := range z.Rules {
|
||||
bts, err = z.Rules[za0001].UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Rules", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *BatchJobExpire) Msgsize() (s int) {
|
||||
s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 16 + z.NotificationCfg.Msgsize() + 6 + z.Retry.Msgsize() + 6 + msgp.ArrayHeaderSize
|
||||
for za0001 := range z.Rules {
|
||||
s += z.Rules[za0001].Msgsize()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "OlderThan":
|
||||
z.OlderThan, err = dc.ReadDuration()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "OlderThan")
|
||||
return
|
||||
}
|
||||
case "CreatedBefore":
|
||||
if dc.IsNil() {
|
||||
err = dc.ReadNil()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CreatedBefore")
|
||||
return
|
||||
}
|
||||
z.CreatedBefore = nil
|
||||
} else {
|
||||
if z.CreatedBefore == nil {
|
||||
z.CreatedBefore = new(time.Time)
|
||||
}
|
||||
*z.CreatedBefore, err = dc.ReadTime()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CreatedBefore")
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Tags":
|
||||
var zb0002 uint32
|
||||
zb0002, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Tags")
|
||||
return
|
||||
}
|
||||
if cap(z.Tags) >= int(zb0002) {
|
||||
z.Tags = (z.Tags)[:zb0002]
|
||||
} else {
|
||||
z.Tags = make([]BatchJobKV, zb0002)
|
||||
}
|
||||
for za0001 := range z.Tags {
|
||||
err = z.Tags[za0001].DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Tags", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Metadata":
|
||||
var zb0003 uint32
|
||||
zb0003, err = dc.ReadArrayHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Metadata")
|
||||
return
|
||||
}
|
||||
if cap(z.Metadata) >= int(zb0003) {
|
||||
z.Metadata = (z.Metadata)[:zb0003]
|
||||
} else {
|
||||
z.Metadata = make([]BatchJobKV, zb0003)
|
||||
}
|
||||
for za0002 := range z.Metadata {
|
||||
err = z.Metadata[za0002].DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Metadata", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Size":
|
||||
err = z.Size.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
case "Type":
|
||||
z.Type, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Type")
|
||||
return
|
||||
}
|
||||
case "Name":
|
||||
z.Name, err = dc.ReadString()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Name")
|
||||
return
|
||||
}
|
||||
case "Purge":
|
||||
var zb0004 uint32
|
||||
zb0004, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge")
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "RetainVersions":
|
||||
z.Purge.RetainVersions, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge", "RetainVersions")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 8
|
||||
// write "OlderThan"
|
||||
err = en.Append(0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteDuration(z.OlderThan)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "OlderThan")
|
||||
return
|
||||
}
|
||||
// write "CreatedBefore"
|
||||
err = en.Append(0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
if z.CreatedBefore == nil {
|
||||
err = en.WriteNil()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
err = en.WriteTime(*z.CreatedBefore)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CreatedBefore")
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "Tags"
|
||||
err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(len(z.Tags)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Tags")
|
||||
return
|
||||
}
|
||||
for za0001 := range z.Tags {
|
||||
err = z.Tags[za0001].EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Tags", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "Metadata"
|
||||
err = en.Append(0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteArrayHeader(uint32(len(z.Metadata)))
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Metadata")
|
||||
return
|
||||
}
|
||||
for za0002 := range z.Metadata {
|
||||
err = z.Metadata[za0002].EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Metadata", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
// write "Size"
|
||||
err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = z.Size.EncodeMsg(en)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
// write "Type"
|
||||
err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Type)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Type")
|
||||
return
|
||||
}
|
||||
// write "Name"
|
||||
err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteString(z.Name)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Name")
|
||||
return
|
||||
}
|
||||
// write "Purge"
|
||||
err = en.Append(0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// map header, size 1
|
||||
// write "RetainVersions"
|
||||
err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.Purge.RetainVersions)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge", "RetainVersions")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 8
|
||||
// string "OlderThan"
|
||||
o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
|
||||
o = msgp.AppendDuration(o, z.OlderThan)
|
||||
// string "CreatedBefore"
|
||||
o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
|
||||
if z.CreatedBefore == nil {
|
||||
o = msgp.AppendNil(o)
|
||||
} else {
|
||||
o = msgp.AppendTime(o, *z.CreatedBefore)
|
||||
}
|
||||
// string "Tags"
|
||||
o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73)
|
||||
o = msgp.AppendArrayHeader(o, uint32(len(z.Tags)))
|
||||
for za0001 := range z.Tags {
|
||||
o, err = z.Tags[za0001].MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Tags", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
// string "Metadata"
|
||||
o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
|
||||
o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata)))
|
||||
for za0002 := range z.Metadata {
|
||||
o, err = z.Metadata[za0002].MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Metadata", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
// string "Size"
|
||||
o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65)
|
||||
o, err = z.Size.MarshalMsg(o)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
// string "Type"
|
||||
o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65)
|
||||
o = msgp.AppendString(o, z.Type)
|
||||
// string "Name"
|
||||
o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
|
||||
o = msgp.AppendString(o, z.Name)
|
||||
// string "Purge"
|
||||
o = append(o, 0xa5, 0x50, 0x75, 0x72, 0x67, 0x65)
|
||||
// map header, size 1
|
||||
// string "RetainVersions"
|
||||
o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||
o = msgp.AppendInt(o, z.Purge.RetainVersions)
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "OlderThan":
|
||||
z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "OlderThan")
|
||||
return
|
||||
}
|
||||
case "CreatedBefore":
|
||||
if msgp.IsNil(bts) {
|
||||
bts, err = msgp.ReadNilBytes(bts)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
z.CreatedBefore = nil
|
||||
} else {
|
||||
if z.CreatedBefore == nil {
|
||||
z.CreatedBefore = new(time.Time)
|
||||
}
|
||||
*z.CreatedBefore, bts, err = msgp.ReadTimeBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "CreatedBefore")
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Tags":
|
||||
var zb0002 uint32
|
||||
zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Tags")
|
||||
return
|
||||
}
|
||||
if cap(z.Tags) >= int(zb0002) {
|
||||
z.Tags = (z.Tags)[:zb0002]
|
||||
} else {
|
||||
z.Tags = make([]BatchJobKV, zb0002)
|
||||
}
|
||||
for za0001 := range z.Tags {
|
||||
bts, err = z.Tags[za0001].UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Tags", za0001)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Metadata":
|
||||
var zb0003 uint32
|
||||
zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Metadata")
|
||||
return
|
||||
}
|
||||
if cap(z.Metadata) >= int(zb0003) {
|
||||
z.Metadata = (z.Metadata)[:zb0003]
|
||||
} else {
|
||||
z.Metadata = make([]BatchJobKV, zb0003)
|
||||
}
|
||||
for za0002 := range z.Metadata {
|
||||
bts, err = z.Metadata[za0002].UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Metadata", za0002)
|
||||
return
|
||||
}
|
||||
}
|
||||
case "Size":
|
||||
bts, err = z.Size.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Size")
|
||||
return
|
||||
}
|
||||
case "Type":
|
||||
z.Type, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Type")
|
||||
return
|
||||
}
|
||||
case "Name":
|
||||
z.Name, bts, err = msgp.ReadStringBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Name")
|
||||
return
|
||||
}
|
||||
case "Purge":
|
||||
var zb0004 uint32
|
||||
zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge")
|
||||
return
|
||||
}
|
||||
for zb0004 > 0 {
|
||||
zb0004--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge")
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "RetainVersions":
|
||||
z.Purge.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge", "RetainVersions")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "Purge")
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z *BatchJobExpireFilter) Msgsize() (s int) {
|
||||
s = 1 + 10 + msgp.DurationSize + 14
|
||||
if z.CreatedBefore == nil {
|
||||
s += msgp.NilSize
|
||||
} else {
|
||||
s += msgp.TimeSize
|
||||
}
|
||||
s += 5 + msgp.ArrayHeaderSize
|
||||
for za0001 := range z.Tags {
|
||||
s += z.Tags[za0001].Msgsize()
|
||||
}
|
||||
s += 9 + msgp.ArrayHeaderSize
|
||||
for za0002 := range z.Metadata {
|
||||
s += z.Metadata[za0002].Msgsize()
|
||||
}
|
||||
s += 5 + z.Size.Msgsize() + 5 + msgp.StringPrefixSize + len(z.Type) + 5 + msgp.StringPrefixSize + len(z.Name) + 6 + 1 + 15 + msgp.IntSize
|
||||
return
|
||||
}
|
||||
|
||||
// DecodeMsg implements msgp.Decodable
|
||||
func (z *BatchJobExpirePurge) DecodeMsg(dc *msgp.Reader) (err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, err = dc.ReadMapHeader()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, err = dc.ReadMapKeyPtr()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "RetainVersions":
|
||||
z.RetainVersions, err = dc.ReadInt()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "RetainVersions")
|
||||
return
|
||||
}
|
||||
default:
|
||||
err = dc.Skip()
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// EncodeMsg implements msgp.Encodable
|
||||
func (z BatchJobExpirePurge) EncodeMsg(en *msgp.Writer) (err error) {
|
||||
// map header, size 1
|
||||
// write "RetainVersions"
|
||||
err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
err = en.WriteInt(z.RetainVersions)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "RetainVersions")
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// MarshalMsg implements msgp.Marshaler
|
||||
func (z BatchJobExpirePurge) MarshalMsg(b []byte) (o []byte, err error) {
|
||||
o = msgp.Require(b, z.Msgsize())
|
||||
// map header, size 1
|
||||
// string "RetainVersions"
|
||||
o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
|
||||
o = msgp.AppendInt(o, z.RetainVersions)
|
||||
return
|
||||
}
|
||||
|
||||
// UnmarshalMsg implements msgp.Unmarshaler
|
||||
func (z *BatchJobExpirePurge) UnmarshalMsg(bts []byte) (o []byte, err error) {
|
||||
var field []byte
|
||||
_ = field
|
||||
var zb0001 uint32
|
||||
zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
for zb0001 > 0 {
|
||||
zb0001--
|
||||
field, bts, err = msgp.ReadMapKeyZC(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
switch msgp.UnsafeString(field) {
|
||||
case "RetainVersions":
|
||||
z.RetainVersions, bts, err = msgp.ReadIntBytes(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err, "RetainVersions")
|
||||
return
|
||||
}
|
||||
default:
|
||||
bts, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
err = msgp.WrapError(err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
o = bts
|
||||
return
|
||||
}
|
||||
|
||||
// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
|
||||
func (z BatchJobExpirePurge) Msgsize() (s int) {
|
||||
s = 1 + 15 + msgp.IntSize
|
||||
return
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
package cmd
|
||||
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
func TestMarshalUnmarshalBatchJobExpire(t *testing.T) {
|
||||
v := BatchJobExpire{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobExpire(b *testing.B) {
|
||||
v := BatchJobExpire{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobExpire(b *testing.B) {
|
||||
v := BatchJobExpire{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobExpire(b *testing.B) {
|
||||
v := BatchJobExpire{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobExpire(t *testing.T) {
|
||||
v := BatchJobExpire{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobExpire Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobExpire{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobExpire(b *testing.B) {
|
||||
v := BatchJobExpire{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobExpire(b *testing.B) {
|
||||
v := BatchJobExpire{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobExpireFilter(t *testing.T) {
|
||||
v := BatchJobExpireFilter{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobExpireFilter(b *testing.B) {
|
||||
v := BatchJobExpireFilter{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobExpireFilter(b *testing.B) {
|
||||
v := BatchJobExpireFilter{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobExpireFilter(b *testing.B) {
|
||||
v := BatchJobExpireFilter{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobExpireFilter(t *testing.T) {
|
||||
v := BatchJobExpireFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobExpireFilter Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobExpireFilter{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobExpireFilter(b *testing.B) {
|
||||
v := BatchJobExpireFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobExpireFilter(b *testing.B) {
|
||||
v := BatchJobExpireFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobExpirePurge(t *testing.T) {
|
||||
v := BatchJobExpirePurge{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobExpirePurge(b *testing.B) {
|
||||
v := BatchJobExpirePurge{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobExpirePurge(b *testing.B) {
|
||||
v := BatchJobExpirePurge{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobExpirePurge(b *testing.B) {
|
||||
v := BatchJobExpirePurge{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobExpirePurge(t *testing.T) {
|
||||
v := BatchJobExpirePurge{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobExpirePurge Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobExpirePurge{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobExpirePurge(b *testing.B) {
|
||||
v := BatchJobExpirePurge{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobExpirePurge(b *testing.B) {
|
||||
v := BatchJobExpirePurge{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
func TestParseBatchJobExpire(t *testing.T) {
|
||||
expireYaml := `
|
||||
expire: # Expire objects that match a condition
|
||||
apiVersion: v1
|
||||
bucket: mybucket # Bucket where this batch job will expire matching objects from
|
||||
prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below.
|
||||
rules:
|
||||
- type: object # regular objects with zero ore more older versions
|
||||
name: NAME # match object names that satisfy the wildcard expression.
|
||||
olderThan: 70h # match objects older than this value
|
||||
createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||
tags:
|
||||
- key: name
|
||||
value: pick* # match objects with tag 'name', all values starting with 'pick'
|
||||
metadata:
|
||||
- key: content-type
|
||||
value: image/* # match objects with 'content-type', all values starting with 'image/'
|
||||
size:
|
||||
lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
|
||||
greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
|
||||
purge:
|
||||
# retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||
# retainVersions: 5 # keep the latest 5 versions of the object.
|
||||
|
||||
- type: deleted # objects with delete marker as their latest version
|
||||
name: NAME # match object names that satisfy the wildcard expression.
|
||||
olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
|
||||
createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
|
||||
purge:
|
||||
# retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
|
||||
# retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
|
||||
|
||||
notify:
|
||||
endpoint: https://notify.endpoint # notification endpoint to receive job completion status
|
||||
token: Bearer xxxxx # optional authentication token for the notification endpoint
|
||||
|
||||
retry:
|
||||
attempts: 10 # number of retries for the job before giving up
|
||||
delay: 500ms # least amount of delay between each retry
|
||||
`
|
||||
var job BatchJobRequest
|
||||
err := yaml.UnmarshalStrict([]byte(expireYaml), &job)
|
||||
if err != nil {
|
||||
t.Fatal("Failed to parse batch-job-expire yaml", err)
|
||||
}
|
||||
}
|
||||
+466
-340
File diff suppressed because it is too large
Load Diff
+72
-2248
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,163 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/pkg/v2/wildcard"
|
||||
)
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
// BatchJobKV is a key-value data type which supports wildcard matching
|
||||
type BatchJobKV struct {
|
||||
Key string `yaml:"key" json:"key"`
|
||||
Value string `yaml:"value" json:"value"`
|
||||
}
|
||||
|
||||
// Validate returns an error if key is empty
|
||||
func (kv BatchJobKV) Validate() error {
|
||||
if kv.Key == "" {
|
||||
return errInvalidArgument
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Empty indicates if kv is not set
|
||||
func (kv BatchJobKV) Empty() bool {
|
||||
return kv.Key == "" && kv.Value == ""
|
||||
}
|
||||
|
||||
// Match matches input kv with kv, value will be wildcard matched depending on the user input
|
||||
func (kv BatchJobKV) Match(ikv BatchJobKV) bool {
|
||||
if kv.Empty() {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(kv.Key, ikv.Key) {
|
||||
return wildcard.Match(kv.Value, ikv.Value)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BatchJobNotification stores notification endpoint and token information.
|
||||
// Used by batch jobs to notify of their status.
|
||||
type BatchJobNotification struct {
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Token string `yaml:"token" json:"token"`
|
||||
}
|
||||
|
||||
// BatchJobRetry stores retry configuration used in the event of failures.
|
||||
type BatchJobRetry struct {
|
||||
Attempts int `yaml:"attempts" json:"attempts"` // number of retry attempts
|
||||
Delay time.Duration `yaml:"delay" json:"delay"` // delay between each retries
|
||||
}
|
||||
|
||||
// Validate validates input replicate retries.
|
||||
func (r BatchJobRetry) Validate() error {
|
||||
if r.Attempts < 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
if r.Delay < 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// # snowball based archive transfer is by default enabled when source
|
||||
// # is local and target is remote which is also minio.
|
||||
// snowball:
|
||||
// disable: false # optionally turn-off snowball archive transfer
|
||||
// batch: 100 # upto this many objects per archive
|
||||
// inmemory: true # indicates if the archive must be staged locally or in-memory
|
||||
// compress: true # S2/Snappy compressed archive
|
||||
// smallerThan: 5MiB # create archive for all objects smaller than 5MiB
|
||||
// skipErrs: false # skips any source side read() errors
|
||||
|
||||
// BatchJobSnowball describes the snowball feature when replicating objects from a local source to a remote target
|
||||
type BatchJobSnowball struct {
|
||||
Disable *bool `yaml:"disable" json:"disable"`
|
||||
Batch *int `yaml:"batch" json:"batch"`
|
||||
InMemory *bool `yaml:"inmemory" json:"inmemory"`
|
||||
Compress *bool `yaml:"compress" json:"compress"`
|
||||
SmallerThan *string `yaml:"smallerThan" json:"smallerThan"`
|
||||
SkipErrs *bool `yaml:"skipErrs" json:"skipErrs"`
|
||||
}
|
||||
|
||||
// Validate the snowball parameters in the job description
|
||||
func (b BatchJobSnowball) Validate() error {
|
||||
if *b.Batch <= 0 {
|
||||
return errors.New("batch number should be non positive zero")
|
||||
}
|
||||
_, err := humanize.ParseBytes(*b.SmallerThan)
|
||||
return err
|
||||
}
|
||||
|
||||
// BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan
|
||||
type BatchJobSizeFilter struct {
|
||||
UpperBound BatchJobSize `yaml:"lessThan" json:"lessThan"`
|
||||
LowerBound BatchJobSize `yaml:"greaterThan" json:"greaterThan"`
|
||||
}
|
||||
|
||||
// InRange returns true in the following cases and false otherwise,
|
||||
// - sf.LowerBound < sz, when sf.LowerBound alone is specified
|
||||
// - sz < sf.UpperBound, when sf.UpperBound alone is specified
|
||||
// - sf.LowerBound < sz < sf.UpperBound when both are specified,
|
||||
func (sf BatchJobSizeFilter) InRange(sz int64) bool {
|
||||
if sf.UpperBound > 0 && sz > int64(sf.UpperBound) {
|
||||
return false
|
||||
}
|
||||
|
||||
if sf.LowerBound > 0 && sz < int64(sf.LowerBound) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var errInvalidBatchJobSizeFilter = errors.New("invalid batch-job size filter")
|
||||
|
||||
// Validate checks if sf is a valid batch-job size filter
|
||||
func (sf BatchJobSizeFilter) Validate() error {
|
||||
if sf.LowerBound > 0 && sf.UpperBound > 0 && sf.LowerBound >= sf.UpperBound {
|
||||
return errInvalidBatchJobSizeFilter
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchJobSize supports humanized byte values in yaml files type BatchJobSize uint64
|
||||
type BatchJobSize int64
|
||||
|
||||
// UnmarshalYAML to parse humanized byte values
|
||||
func (s *BatchJobSize) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||
var batchExpireSz string
|
||||
err := unmarshal(&batchExpireSz)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sz, err := humanize.ParseBytes(batchExpireSz)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
*s = BatchJobSize(sz)
|
||||
return nil
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,575 @@
|
||||
package cmd
|
||||
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
func TestMarshalUnmarshalBatchJobKV(t *testing.T) {
|
||||
v := BatchJobKV{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobKV(b *testing.B) {
|
||||
v := BatchJobKV{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobKV(b *testing.B) {
|
||||
v := BatchJobKV{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobKV(b *testing.B) {
|
||||
v := BatchJobKV{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobKV(t *testing.T) {
|
||||
v := BatchJobKV{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobKV Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobKV{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobKV(b *testing.B) {
|
||||
v := BatchJobKV{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobKV(b *testing.B) {
|
||||
v := BatchJobKV{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobNotification(t *testing.T) {
|
||||
v := BatchJobNotification{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobNotification(b *testing.B) {
|
||||
v := BatchJobNotification{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobNotification(b *testing.B) {
|
||||
v := BatchJobNotification{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobNotification(b *testing.B) {
|
||||
v := BatchJobNotification{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobNotification(t *testing.T) {
|
||||
v := BatchJobNotification{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobNotification Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobNotification{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobNotification(b *testing.B) {
|
||||
v := BatchJobNotification{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobNotification(b *testing.B) {
|
||||
v := BatchJobNotification{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobRetry(t *testing.T) {
|
||||
v := BatchJobRetry{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobRetry(b *testing.B) {
|
||||
v := BatchJobRetry{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobRetry(b *testing.B) {
|
||||
v := BatchJobRetry{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobRetry(b *testing.B) {
|
||||
v := BatchJobRetry{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobRetry(t *testing.T) {
|
||||
v := BatchJobRetry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobRetry Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobRetry{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobRetry(b *testing.B) {
|
||||
v := BatchJobRetry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobRetry(b *testing.B) {
|
||||
v := BatchJobRetry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobSizeFilter(t *testing.T) {
|
||||
v := BatchJobSizeFilter{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobSizeFilter(b *testing.B) {
|
||||
v := BatchJobSizeFilter{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobSizeFilter(b *testing.B) {
|
||||
v := BatchJobSizeFilter{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobSizeFilter(b *testing.B) {
|
||||
v := BatchJobSizeFilter{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobSizeFilter(t *testing.T) {
|
||||
v := BatchJobSizeFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobSizeFilter Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobSizeFilter{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobSizeFilter(b *testing.B) {
|
||||
v := BatchJobSizeFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobSizeFilter(b *testing.B) {
|
||||
v := BatchJobSizeFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobSnowball(t *testing.T) {
|
||||
v := BatchJobSnowball{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobSnowball(b *testing.B) {
|
||||
v := BatchJobSnowball{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobSnowball(b *testing.B) {
|
||||
v := BatchJobSnowball{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobSnowball(b *testing.B) {
|
||||
v := BatchJobSnowball{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobSnowball(t *testing.T) {
|
||||
v := BatchJobSnowball{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobSnowball Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobSnowball{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobSnowball(b *testing.B) {
|
||||
v := BatchJobSnowball{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobSnowball(b *testing.B) {
|
||||
v := BatchJobSnowball{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBatchJobSizeInRange(t *testing.T) {
|
||||
tests := []struct {
|
||||
objSize int64
|
||||
sizeFilter BatchJobSizeFilter
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
// 1Mib < 2Mib < 10MiB -> in range
|
||||
objSize: 2 << 20,
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 10 << 20,
|
||||
LowerBound: 1 << 20,
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// 2KiB < 1 MiB -> out of range from left
|
||||
objSize: 2 << 10,
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 10 << 20,
|
||||
LowerBound: 1 << 20,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// 11MiB > 10 MiB -> out of range from right
|
||||
objSize: 11 << 20,
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 10 << 20,
|
||||
LowerBound: 1 << 20,
|
||||
},
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
// 2MiB < 10MiB -> in range
|
||||
objSize: 2 << 20,
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 10 << 20,
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
// 2MiB > 1MiB -> in range
|
||||
objSize: 2 << 20,
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
LowerBound: 1 << 20,
|
||||
},
|
||||
want: true,
|
||||
},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
|
||||
if got := test.sizeFilter.InRange(test.objSize); got != test.want {
|
||||
t.Fatalf("Expected %v but got %v", test.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBatchJobSizeValidate(t *testing.T) {
|
||||
tests := []struct {
|
||||
sizeFilter BatchJobSizeFilter
|
||||
err error
|
||||
}{
|
||||
{
|
||||
// Unspecified size filter is a valid filter
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 0,
|
||||
LowerBound: 0,
|
||||
},
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 0,
|
||||
LowerBound: 1 << 20,
|
||||
},
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 10 << 20,
|
||||
LowerBound: 0,
|
||||
},
|
||||
err: nil,
|
||||
},
|
||||
{
|
||||
// LowerBound > UpperBound -> empty range
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 1 << 20,
|
||||
LowerBound: 10 << 20,
|
||||
},
|
||||
err: errInvalidBatchJobSizeFilter,
|
||||
},
|
||||
{
|
||||
// LowerBound == UpperBound -> empty range
|
||||
sizeFilter: BatchJobSizeFilter{
|
||||
UpperBound: 1 << 20,
|
||||
LowerBound: 1 << 20,
|
||||
},
|
||||
err: errInvalidBatchJobSizeFilter,
|
||||
},
|
||||
}
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) {
|
||||
if err := test.sizeFilter.Validate(); err != test.err {
|
||||
t.Fatalf("Expected %v but got %v", test.err, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
miniogo "github.com/minio/minio-go/v7"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
)
|
||||
|
||||
//go:generate msgp -file $GOFILE
|
||||
|
||||
// replicate:
|
||||
// # source of the objects to be replicated
|
||||
// source:
|
||||
// type: "minio"
|
||||
// bucket: "testbucket"
|
||||
// prefix: "spark/"
|
||||
//
|
||||
// # optional flags based filtering criteria
|
||||
// # for source objects
|
||||
// flags:
|
||||
// filter:
|
||||
// newerThan: "7d"
|
||||
// olderThan: "7d"
|
||||
// createdAfter: "date"
|
||||
// createdBefore: "date"
|
||||
// tags:
|
||||
// - key: "name"
|
||||
// value: "value*"
|
||||
// metadata:
|
||||
// - key: "content-type"
|
||||
// value: "image/*"
|
||||
// notify:
|
||||
// endpoint: "https://splunk-hec.dev.com"
|
||||
// token: "Splunk ..." # e.g. "Bearer token"
|
||||
//
|
||||
// # target where the objects must be replicated
|
||||
// target:
|
||||
// type: "minio"
|
||||
// bucket: "testbucket1"
|
||||
// endpoint: "https://play.min.io"
|
||||
// path: "on"
|
||||
// credentials:
|
||||
// accessKey: "minioadmin"
|
||||
// secretKey: "minioadmin"
|
||||
// sessionToken: ""
|
||||
|
||||
// BatchReplicateFilter holds all the filters currently supported for batch replication
|
||||
type BatchReplicateFilter struct {
|
||||
NewerThan time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
|
||||
OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
|
||||
CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"`
|
||||
CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"`
|
||||
Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"`
|
||||
Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"`
|
||||
}
|
||||
|
||||
// BatchJobReplicateFlags various configurations for replication job definition currently includes
|
||||
// - filter
|
||||
// - notify
|
||||
// - retry
|
||||
type BatchJobReplicateFlags struct {
|
||||
Filter BatchReplicateFilter `yaml:"filter" json:"filter"`
|
||||
Notify BatchJobNotification `yaml:"notify" json:"notify"`
|
||||
Retry BatchJobRetry `yaml:"retry" json:"retry"`
|
||||
}
|
||||
|
||||
// BatchJobReplicateResourceType defines the type of batch jobs
|
||||
type BatchJobReplicateResourceType string
|
||||
|
||||
// Validate validates if the replicate resource type is recognized and supported
|
||||
func (t BatchJobReplicateResourceType) Validate() error {
|
||||
switch t {
|
||||
case BatchJobReplicateResourceMinIO:
|
||||
case BatchJobReplicateResourceS3:
|
||||
default:
|
||||
return errInvalidArgument
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (t BatchJobReplicateResourceType) isMinio() bool {
|
||||
return t == BatchJobReplicateResourceMinIO
|
||||
}
|
||||
|
||||
// Different types of batch jobs..
|
||||
const (
|
||||
BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
|
||||
BatchJobReplicateResourceS3 BatchJobReplicateResourceType = "s3"
|
||||
|
||||
// add future targets
|
||||
)
|
||||
|
||||
// BatchJobReplicateCredentials access credentials for batch replication it may
|
||||
// be either for target or source.
|
||||
type BatchJobReplicateCredentials struct {
|
||||
AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"`
|
||||
SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"`
|
||||
SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
|
||||
}
|
||||
|
||||
// Empty indicates if credentials are not set
|
||||
func (c BatchJobReplicateCredentials) Empty() bool {
|
||||
return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
|
||||
}
|
||||
|
||||
// Validate validates if credentials are valid
|
||||
func (c BatchJobReplicateCredentials) Validate() error {
|
||||
if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
|
||||
return errInvalidArgument
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchJobReplicateTarget describes target element of the replication job that receives
|
||||
// the filtered data from source
|
||||
type BatchJobReplicateTarget struct {
|
||||
Type BatchJobReplicateResourceType `yaml:"type" json:"type"`
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
Prefix string `yaml:"prefix" json:"prefix"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Path string `yaml:"path" json:"path"`
|
||||
Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"`
|
||||
}
|
||||
|
||||
// ValidPath returns true if path is valid
|
||||
func (t BatchJobReplicateTarget) ValidPath() bool {
|
||||
return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
|
||||
}
|
||||
|
||||
// BatchJobReplicateSource describes source element of the replication job that is
|
||||
// the source of the data for the target
|
||||
type BatchJobReplicateSource struct {
|
||||
Type BatchJobReplicateResourceType `yaml:"type" json:"type"`
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
Prefix string `yaml:"prefix" json:"prefix"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Path string `yaml:"path" json:"path"`
|
||||
Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"`
|
||||
Snowball BatchJobSnowball `yaml:"snowball" json:"snowball"`
|
||||
}
|
||||
|
||||
// ValidPath returns true if path is valid
|
||||
func (s BatchJobReplicateSource) ValidPath() bool {
|
||||
switch s.Path {
|
||||
case "on", "off", "auto", "":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// BatchJobReplicateV1 v1 of batch job replication
|
||||
type BatchJobReplicateV1 struct {
|
||||
APIVersion string `yaml:"apiVersion" json:"apiVersion"`
|
||||
Flags BatchJobReplicateFlags `yaml:"flags" json:"flags"`
|
||||
Target BatchJobReplicateTarget `yaml:"target" json:"target"`
|
||||
Source BatchJobReplicateSource `yaml:"source" json:"source"`
|
||||
|
||||
clnt *miniogo.Core `msg:"-"`
|
||||
}
|
||||
|
||||
// RemoteToLocal returns true if source is remote and target is local
|
||||
func (r BatchJobReplicateV1) RemoteToLocal() bool {
|
||||
return !r.Source.Creds.Empty()
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,688 @@
|
||||
package cmd
|
||||
|
||||
// Code generated by github.com/tinylib/msgp DO NOT EDIT.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
)
|
||||
|
||||
func TestMarshalUnmarshalBatchJobReplicateCredentials(t *testing.T) {
|
||||
v := BatchJobReplicateCredentials{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobReplicateCredentials(b *testing.B) {
|
||||
v := BatchJobReplicateCredentials{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobReplicateCredentials(b *testing.B) {
|
||||
v := BatchJobReplicateCredentials{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobReplicateCredentials(b *testing.B) {
|
||||
v := BatchJobReplicateCredentials{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobReplicateCredentials(t *testing.T) {
|
||||
v := BatchJobReplicateCredentials{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobReplicateCredentials Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobReplicateCredentials{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobReplicateCredentials(b *testing.B) {
|
||||
v := BatchJobReplicateCredentials{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobReplicateCredentials(b *testing.B) {
|
||||
v := BatchJobReplicateCredentials{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobReplicateFlags(t *testing.T) {
|
||||
v := BatchJobReplicateFlags{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobReplicateFlags(b *testing.B) {
|
||||
v := BatchJobReplicateFlags{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobReplicateFlags(b *testing.B) {
|
||||
v := BatchJobReplicateFlags{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobReplicateFlags(b *testing.B) {
|
||||
v := BatchJobReplicateFlags{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobReplicateFlags(t *testing.T) {
|
||||
v := BatchJobReplicateFlags{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobReplicateFlags Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobReplicateFlags{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobReplicateFlags(b *testing.B) {
|
||||
v := BatchJobReplicateFlags{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobReplicateFlags(b *testing.B) {
|
||||
v := BatchJobReplicateFlags{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobReplicateSource(t *testing.T) {
|
||||
v := BatchJobReplicateSource{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobReplicateSource(b *testing.B) {
|
||||
v := BatchJobReplicateSource{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobReplicateSource(b *testing.B) {
|
||||
v := BatchJobReplicateSource{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobReplicateSource(b *testing.B) {
|
||||
v := BatchJobReplicateSource{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobReplicateSource(t *testing.T) {
|
||||
v := BatchJobReplicateSource{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobReplicateSource Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobReplicateSource{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobReplicateSource(b *testing.B) {
|
||||
v := BatchJobReplicateSource{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobReplicateSource(b *testing.B) {
|
||||
v := BatchJobReplicateSource{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobReplicateTarget(t *testing.T) {
|
||||
v := BatchJobReplicateTarget{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobReplicateTarget(b *testing.B) {
|
||||
v := BatchJobReplicateTarget{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobReplicateTarget(b *testing.B) {
|
||||
v := BatchJobReplicateTarget{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobReplicateTarget(b *testing.B) {
|
||||
v := BatchJobReplicateTarget{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobReplicateTarget(t *testing.T) {
|
||||
v := BatchJobReplicateTarget{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobReplicateTarget Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobReplicateTarget{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobReplicateTarget(b *testing.B) {
|
||||
v := BatchJobReplicateTarget{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobReplicateTarget(b *testing.B) {
|
||||
v := BatchJobReplicateTarget{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchJobReplicateV1(t *testing.T) {
|
||||
v := BatchJobReplicateV1{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchJobReplicateV1(b *testing.B) {
|
||||
v := BatchJobReplicateV1{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchJobReplicateV1(b *testing.B) {
|
||||
v := BatchJobReplicateV1{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchJobReplicateV1(b *testing.B) {
|
||||
v := BatchJobReplicateV1{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchJobReplicateV1(t *testing.T) {
|
||||
v := BatchJobReplicateV1{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchJobReplicateV1 Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchJobReplicateV1{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchJobReplicateV1(b *testing.B) {
|
||||
v := BatchJobReplicateV1{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchJobReplicateV1(b *testing.B) {
|
||||
v := BatchJobReplicateV1{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchReplicateFilter(t *testing.T) {
|
||||
v := BatchReplicateFilter{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchReplicateFilter(b *testing.B) {
|
||||
v := BatchReplicateFilter{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchReplicateFilter(b *testing.B) {
|
||||
v := BatchReplicateFilter{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchReplicateFilter(b *testing.B) {
|
||||
v := BatchReplicateFilter{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchReplicateFilter(t *testing.T) {
|
||||
v := BatchReplicateFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchReplicateFilter Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchReplicateFilter{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchReplicateFilter(b *testing.B) {
|
||||
v := BatchReplicateFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchReplicateFilter(b *testing.B) {
|
||||
v := BatchReplicateFilter{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
-116
@@ -18,13 +18,9 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"runtime"
|
||||
@@ -38,9 +34,8 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/env"
|
||||
"github.com/minio/pkg/wildcard"
|
||||
"github.com/minio/pkg/workers"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
"github.com/minio/pkg/v2/workers"
|
||||
)
|
||||
|
||||
// keyrotate:
|
||||
@@ -76,56 +71,6 @@ import (
|
||||
|
||||
//go:generate msgp -file $GOFILE -unexported
|
||||
|
||||
// BatchKeyRotateKV is a datatype that holds key and values for filtering of objects
|
||||
// used by metadata filter as well as tags based filtering.
|
||||
type BatchKeyRotateKV struct {
|
||||
Key string `yaml:"key" json:"key"`
|
||||
Value string `yaml:"value" json:"value"`
|
||||
}
|
||||
|
||||
// Validate returns an error if key is empty
|
||||
func (kv BatchKeyRotateKV) Validate() error {
|
||||
if kv.Key == "" {
|
||||
return errInvalidArgument
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Empty indicates if kv is not set
|
||||
func (kv BatchKeyRotateKV) Empty() bool {
|
||||
return kv.Key == "" && kv.Value == ""
|
||||
}
|
||||
|
||||
// Match matches input kv with kv, value will be wildcard matched depending on the user input
|
||||
func (kv BatchKeyRotateKV) Match(ikv BatchKeyRotateKV) bool {
|
||||
if kv.Empty() {
|
||||
return true
|
||||
}
|
||||
if strings.EqualFold(kv.Key, ikv.Key) {
|
||||
return wildcard.Match(kv.Value, ikv.Value)
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// BatchKeyRotateRetry datatype represents total retry attempts and delay between each retries.
|
||||
type BatchKeyRotateRetry struct {
|
||||
Attempts int `yaml:"attempts" json:"attempts"` // number of retry attempts
|
||||
Delay time.Duration `yaml:"delay" json:"delay"` // delay between each retries
|
||||
}
|
||||
|
||||
// Validate validates input replicate retries.
|
||||
func (r BatchKeyRotateRetry) Validate() error {
|
||||
if r.Attempts < 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
if r.Delay < 0 {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// BatchKeyRotationType defines key rotation type
|
||||
type BatchKeyRotationType string
|
||||
|
||||
@@ -178,13 +123,13 @@ func (e BatchJobKeyRotateEncryption) Validate() error {
|
||||
|
||||
// BatchKeyRotateFilter holds all the filters currently supported for batch replication
|
||||
type BatchKeyRotateFilter struct {
|
||||
NewerThan time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
|
||||
OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
|
||||
CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"`
|
||||
CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"`
|
||||
Tags []BatchKeyRotateKV `yaml:"tags,omitempty" json:"tags"`
|
||||
Metadata []BatchKeyRotateKV `yaml:"metadata,omitempty" json:"metadata"`
|
||||
KMSKeyID string `yaml:"kmskeyid" json:"kmskey"`
|
||||
NewerThan time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
|
||||
OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
|
||||
CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"`
|
||||
CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"`
|
||||
Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"`
|
||||
Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"`
|
||||
KMSKeyID string `yaml:"kmskeyid" json:"kmskey"`
|
||||
}
|
||||
|
||||
// BatchKeyRotateNotification success or failure notification endpoint for each job attempts
|
||||
@@ -198,9 +143,9 @@ type BatchKeyRotateNotification struct {
|
||||
// - notify
|
||||
// - retry
|
||||
type BatchJobKeyRotateFlags struct {
|
||||
Filter BatchKeyRotateFilter `yaml:"filter" json:"filter"`
|
||||
Notify BatchKeyRotateNotification `yaml:"notify" json:"notify"`
|
||||
Retry BatchKeyRotateRetry `yaml:"retry" json:"retry"`
|
||||
Filter BatchKeyRotateFilter `yaml:"filter" json:"filter"`
|
||||
Notify BatchJobNotification `yaml:"notify" json:"notify"`
|
||||
Retry BatchJobRetry `yaml:"retry" json:"retry"`
|
||||
}
|
||||
|
||||
// BatchJobKeyRotateV1 v1 of batch key rotation job
|
||||
@@ -209,40 +154,12 @@ type BatchJobKeyRotateV1 struct {
|
||||
Flags BatchJobKeyRotateFlags `yaml:"flags" json:"flags"`
|
||||
Bucket string `yaml:"bucket" json:"bucket"`
|
||||
Prefix string `yaml:"prefix" json:"prefix"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"`
|
||||
}
|
||||
|
||||
// Notify notifies notification endpoint if configured regarding job failure or success.
|
||||
func (r BatchJobKeyRotateV1) Notify(ctx context.Context, body io.Reader) error {
|
||||
if r.Flags.Notify.Endpoint == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if r.Flags.Notify.Token != "" {
|
||||
req.Header.Set("Authorization", r.Flags.Notify.Token)
|
||||
}
|
||||
|
||||
clnt := http.Client{Transport: getRemoteInstanceTransport}
|
||||
resp, err := clnt.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
xhttp.DrainBody(resp.Body)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return errors.New(resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error {
|
||||
return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
|
||||
}
|
||||
|
||||
// KeyRotate rotates encryption key of an object
|
||||
@@ -330,7 +247,7 @@ const (
|
||||
batchKeyRotateVersion = batchKeyRotateVersionV1
|
||||
batchKeyRotateAPIVersion = "v1"
|
||||
batchKeyRotateJobDefaultRetries = 3
|
||||
batchKeyRotateJobDefaultRetryDelay = 250 * time.Millisecond
|
||||
batchKeyRotateJobDefaultRetryDelay = 25 * time.Millisecond
|
||||
)
|
||||
|
||||
// Start the batch key rottion job, resumes if there was a pending job via "job.ID"
|
||||
@@ -343,6 +260,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
if err := ri.load(ctx, api, job); err != nil {
|
||||
return err
|
||||
}
|
||||
if ri.Complete {
|
||||
return nil
|
||||
}
|
||||
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
lastObject := ri.Object
|
||||
@@ -351,6 +271,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
if delay == 0 {
|
||||
delay = batchKeyRotateJobDefaultRetryDelay
|
||||
}
|
||||
|
||||
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
skip := func(info FileInfo) (ok bool) {
|
||||
@@ -388,7 +309,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
|
||||
for _, kv := range r.Flags.Filter.Tags {
|
||||
for t, v := range tagMap {
|
||||
if kv.Match(BatchKeyRotateKV{Key: t, Value: v}) {
|
||||
if kv.Match(BatchJobKV{Key: t, Value: v}) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -405,7 +326,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
continue
|
||||
}
|
||||
// We only need to match x-amz-meta or standardHeaders
|
||||
if kv.Match(BatchKeyRotateKV{Key: k, Value: v}) {
|
||||
if kv.Match(BatchJobKV{Key: k, Value: v}) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
@@ -437,9 +358,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
|
||||
results := make(chan ObjectInfo, 100)
|
||||
if err := api.Walk(ctx, r.Bucket, r.Prefix, results, ObjectOptions{
|
||||
WalkMarker: lastObject,
|
||||
WalkFilter: skip,
|
||||
if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
|
||||
Marker: lastObject,
|
||||
Filter: skip,
|
||||
}); err != nil {
|
||||
cancel()
|
||||
// Do not need to retry if we can't list objects on source.
|
||||
@@ -458,14 +379,14 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
defer wk.Give()
|
||||
for attempts := 1; attempts <= retryAttempts; attempts++ {
|
||||
attempts := attempts
|
||||
stopFn := globalBatchJobsMetrics.trace(batchKeyRotationMetricObject, job.ID, attempts, result)
|
||||
stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts)
|
||||
success := true
|
||||
if err := r.KeyRotate(ctx, api, result); err != nil {
|
||||
stopFn(err)
|
||||
stopFn(result, err)
|
||||
logger.LogIf(ctx, err)
|
||||
success = false
|
||||
} else {
|
||||
stopFn(nil)
|
||||
stopFn(result, nil)
|
||||
}
|
||||
ri.trackCurrentBucketObject(r.Bucket, result, success)
|
||||
ri.RetryAttempts = attempts
|
||||
@@ -475,6 +396,13 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
if success {
|
||||
break
|
||||
}
|
||||
if delay > 0 {
|
||||
time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
|
||||
}
|
||||
}
|
||||
|
||||
if wait := globalBatchConfig.KeyRotationWait(); wait > 0 {
|
||||
time.Sleep(wait)
|
||||
}
|
||||
}()
|
||||
}
|
||||
@@ -486,20 +414,11 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
// persist in-memory state to disk.
|
||||
logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
|
||||
|
||||
buf, _ := json.Marshal(ri)
|
||||
if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
|
||||
if err := r.Notify(ctx, ri); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
|
||||
}
|
||||
|
||||
cancel()
|
||||
if ri.Failed {
|
||||
ri.ObjectsFailed = 0
|
||||
ri.Bucket = ""
|
||||
ri.Object = ""
|
||||
ri.Objects = 0
|
||||
time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
+175
-647
File diff suppressed because it is too large
Load Diff
@@ -461,119 +461,6 @@ func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchKeyRotateKV(t *testing.T) {
|
||||
v := BatchKeyRotateKV{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchKeyRotateKV(b *testing.B) {
|
||||
v := BatchKeyRotateKV{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchKeyRotateKV(b *testing.B) {
|
||||
v := BatchKeyRotateKV{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchKeyRotateKV(b *testing.B) {
|
||||
v := BatchKeyRotateKV{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchKeyRotateKV(t *testing.T) {
|
||||
v := BatchKeyRotateKV{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchKeyRotateKV Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchKeyRotateKV{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchKeyRotateKV(b *testing.B) {
|
||||
v := BatchKeyRotateKV{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchKeyRotateKV(b *testing.B) {
|
||||
v := BatchKeyRotateKV{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) {
|
||||
v := BatchKeyRotateNotification{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
@@ -686,116 +573,3 @@ func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMarshalUnmarshalBatchKeyRotateRetry(t *testing.T) {
|
||||
v := BatchKeyRotateRetry{}
|
||||
bts, err := v.MarshalMsg(nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
left, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
|
||||
}
|
||||
|
||||
left, err = msgp.Skip(bts)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(left) > 0 {
|
||||
t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkMarshalMsgBatchKeyRotateRetry(b *testing.B) {
|
||||
v := BatchKeyRotateRetry{}
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.MarshalMsg(nil)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkAppendMsgBatchKeyRotateRetry(b *testing.B) {
|
||||
v := BatchKeyRotateRetry{}
|
||||
bts := make([]byte, 0, v.Msgsize())
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
bts, _ = v.MarshalMsg(bts[0:0])
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkUnmarshalBatchKeyRotateRetry(b *testing.B) {
|
||||
v := BatchKeyRotateRetry{}
|
||||
bts, _ := v.MarshalMsg(nil)
|
||||
b.ReportAllocs()
|
||||
b.SetBytes(int64(len(bts)))
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
_, err := v.UnmarshalMsg(bts)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeDecodeBatchKeyRotateRetry(t *testing.T) {
|
||||
v := BatchKeyRotateRetry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
|
||||
m := v.Msgsize()
|
||||
if buf.Len() > m {
|
||||
t.Log("WARNING: TestEncodeDecodeBatchKeyRotateRetry Msgsize() is inaccurate")
|
||||
}
|
||||
|
||||
vn := BatchKeyRotateRetry{}
|
||||
err := msgp.Decode(&buf, &vn)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
|
||||
buf.Reset()
|
||||
msgp.Encode(&buf, &v)
|
||||
err = msgp.NewReader(&buf).Skip()
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkEncodeBatchKeyRotateRetry(b *testing.B) {
|
||||
v := BatchKeyRotateRetry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
en := msgp.NewWriter(msgp.Nowhere)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
v.EncodeMsg(en)
|
||||
}
|
||||
en.Flush()
|
||||
}
|
||||
|
||||
func BenchmarkDecodeBatchKeyRotateRetry(b *testing.B) {
|
||||
v := BatchKeyRotateRetry{}
|
||||
var buf bytes.Buffer
|
||||
msgp.Encode(&buf, &v)
|
||||
b.SetBytes(int64(buf.Len()))
|
||||
rd := msgp.NewEndlessReader(buf.Bytes(), b)
|
||||
dc := msgp.NewReader(rd)
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
err := v.DecodeMsg(dc)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,13 +8,14 @@ func _() {
|
||||
// An "invalid array index" compiler error signifies that the constant values have changed.
|
||||
// Re-run the stringer command to generate them again.
|
||||
var x [1]struct{}
|
||||
_ = x[batchReplicationMetricObject-0]
|
||||
_ = x[batchKeyRotationMetricObject-1]
|
||||
_ = x[batchJobMetricReplication-0]
|
||||
_ = x[batchJobMetricKeyRotation-1]
|
||||
_ = x[batchJobMetricExpire-2]
|
||||
}
|
||||
|
||||
const _batchJobMetric_name = "batchReplicationMetricObjectbatchKeyRotationMetricObject"
|
||||
const _batchJobMetric_name = "ReplicationKeyRotationExpire"
|
||||
|
||||
var _batchJobMetric_index = [...]uint8{0, 28, 56}
|
||||
var _batchJobMetric_index = [...]uint8{0, 11, 22, 28}
|
||||
|
||||
func (i batchJobMetric) String() string {
|
||||
if i >= batchJobMetric(len(_batchJobMetric_index)-1) {
|
||||
|
||||
@@ -95,7 +95,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
|
||||
h := algo.New()
|
||||
|
||||
bw := &streamingBitrotWriter{
|
||||
iow: ioutil.NewDeadlineWriter(w, diskMaxTimeout),
|
||||
iow: ioutil.NewDeadlineWriter(w, globalDriveConfig.GetMaxTimeout()),
|
||||
closeWithErr: w.CloseWithError,
|
||||
h: h,
|
||||
shardSize: shardSize,
|
||||
|
||||
+18
-65
@@ -19,98 +19,51 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/pubsub"
|
||||
)
|
||||
|
||||
const bootstrapMsgsLimit = 4 << 10
|
||||
const bootstrapTraceLimit = 4 << 10
|
||||
|
||||
type bootstrapInfo struct {
|
||||
msg string
|
||||
ts time.Time
|
||||
source string
|
||||
}
|
||||
type bootstrapTracer struct {
|
||||
mu sync.RWMutex
|
||||
idx int
|
||||
info [bootstrapMsgsLimit]bootstrapInfo
|
||||
lastUpdate time.Time
|
||||
mu sync.RWMutex
|
||||
info []madmin.TraceInfo
|
||||
}
|
||||
|
||||
var globalBootstrapTracer = &bootstrapTracer{}
|
||||
|
||||
func (bs *bootstrapTracer) DropEvents() {
|
||||
func (bs *bootstrapTracer) Record(info madmin.TraceInfo) {
|
||||
bs.mu.Lock()
|
||||
defer bs.mu.Unlock()
|
||||
|
||||
if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour {
|
||||
bs.info = [4096]bootstrapInfo{}
|
||||
bs.idx = 0
|
||||
if len(bs.info) > bootstrapTraceLimit {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (bs *bootstrapTracer) Empty() bool {
|
||||
var empty bool
|
||||
bs.mu.RLock()
|
||||
empty = bs.info[0].msg == ""
|
||||
bs.mu.RUnlock()
|
||||
|
||||
return empty
|
||||
}
|
||||
|
||||
func (bs *bootstrapTracer) Record(msg string, skip int) {
|
||||
source := getSource(skip + 1)
|
||||
bs.mu.Lock()
|
||||
now := time.Now().UTC()
|
||||
bs.info[bs.idx] = bootstrapInfo{
|
||||
msg: msg,
|
||||
ts: now,
|
||||
source: source,
|
||||
}
|
||||
bs.lastUpdate = now
|
||||
bs.idx = (bs.idx + 1) % bootstrapMsgsLimit
|
||||
bs.mu.Unlock()
|
||||
bs.info = append(bs.info, info)
|
||||
}
|
||||
|
||||
func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
|
||||
traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit)
|
||||
|
||||
// Add all messages in order
|
||||
addAll := func(info []bootstrapInfo) {
|
||||
for _, msg := range info {
|
||||
if msg.ts.IsZero() {
|
||||
continue // skip empty events
|
||||
}
|
||||
traceInfo = append(traceInfo, madmin.TraceInfo{
|
||||
TraceType: madmin.TraceBootstrap,
|
||||
Time: msg.ts,
|
||||
NodeName: globalLocalNodeName,
|
||||
FuncName: "BOOTSTRAP",
|
||||
Message: fmt.Sprintf("%s %s", msg.source, msg.msg),
|
||||
})
|
||||
}
|
||||
}
|
||||
traceInfo := make([]madmin.TraceInfo, 0, bootstrapTraceLimit)
|
||||
|
||||
bs.mu.RLock()
|
||||
addAll(bs.info[bs.idx:])
|
||||
addAll(bs.info[:bs.idx])
|
||||
for _, i := range bs.info {
|
||||
traceInfo = append(traceInfo, i)
|
||||
}
|
||||
bs.mu.RUnlock()
|
||||
|
||||
return traceInfo
|
||||
}
|
||||
|
||||
func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
|
||||
if bs.Empty() {
|
||||
return
|
||||
}
|
||||
for _, bsEvent := range bs.Events() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
default:
|
||||
trace.Publish(bsEvent)
|
||||
if bsEvent.Message != "" {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
default:
|
||||
trace.Publish(bsEvent)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
// Copyright (c) 2015-2023 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestBootstrap(t *testing.T) {
|
||||
// Bootstrap events exceed bootstrap messages limit
|
||||
bsTracer := &bootstrapTracer{}
|
||||
for i := 0; i < bootstrapMsgsLimit+10; i++ {
|
||||
bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
|
||||
}
|
||||
|
||||
traceInfos := bsTracer.Events()
|
||||
if len(traceInfos) != bootstrapMsgsLimit {
|
||||
t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos))
|
||||
}
|
||||
|
||||
// Simulate the case where bootstrap events were updated a day ago
|
||||
bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour)
|
||||
bsTracer.DropEvents()
|
||||
if !bsTracer.Empty() {
|
||||
t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events()))
|
||||
}
|
||||
|
||||
// Fewer than 4K bootstrap events
|
||||
for i := 0; i < 10; i++ {
|
||||
bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
|
||||
}
|
||||
events := bsTracer.Events()
|
||||
if len(events) != 10 {
|
||||
t.Fatalf("Expected length of events %d but got %d", 10, len(events))
|
||||
}
|
||||
for i, traceInfo := range bsTracer.Events() {
|
||||
msg := fmt.Sprintf("msg-%d", i)
|
||||
if !strings.HasSuffix(traceInfo.Message, msg) {
|
||||
t.Fatalf("Expected %s but got %s", msg, traceInfo.Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,12 +20,14 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/set"
|
||||
@@ -33,7 +35,7 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/rest"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/env"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -82,23 +84,40 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
|
||||
ep.Platform, s2.MinioEndpoints[i].Platform)
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
|
||||
var missing []string
|
||||
var mismatching []string
|
||||
for k, v := range s1.MinioEnv {
|
||||
ev, ok := s2.MinioEnv[k]
|
||||
if !ok {
|
||||
missing = append(missing, k)
|
||||
} else if v != ev {
|
||||
mismatching = append(mismatching, k)
|
||||
}
|
||||
}
|
||||
if len(mismatching) > 0 {
|
||||
return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching)
|
||||
}
|
||||
return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing)
|
||||
if reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
|
||||
// Report differences in environment variables.
|
||||
var missing []string
|
||||
var mismatching []string
|
||||
for k, v := range s1.MinioEnv {
|
||||
ev, ok := s2.MinioEnv[k]
|
||||
if !ok {
|
||||
missing = append(missing, k)
|
||||
} else if v != ev {
|
||||
mismatching = append(mismatching, k)
|
||||
}
|
||||
}
|
||||
var extra []string
|
||||
for k := range s2.MinioEnv {
|
||||
_, ok := s1.MinioEnv[k]
|
||||
if !ok {
|
||||
extra = append(extra, k)
|
||||
}
|
||||
}
|
||||
msg := "Expected same MINIO_ environment variables and values across all servers: "
|
||||
if len(missing) > 0 {
|
||||
msg += fmt.Sprintf(`Missing environment values: %v. `, missing)
|
||||
}
|
||||
if len(mismatching) > 0 {
|
||||
msg += fmt.Sprintf(`Mismatching environment values: %v. `, mismatching)
|
||||
}
|
||||
if len(extra) > 0 {
|
||||
msg += fmt.Sprintf(`Extra environment values: %v. `, extra)
|
||||
}
|
||||
|
||||
return errors.New(strings.TrimSpace(msg))
|
||||
}
|
||||
|
||||
var skipEnvs = map[string]struct{}{
|
||||
@@ -221,7 +240,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
|
||||
for onlineServers < len(clnts)/2 {
|
||||
for _, clnt := range clnts {
|
||||
if err := clnt.Verify(ctx, srcCfg); err != nil {
|
||||
bootstrapTrace(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
|
||||
bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
|
||||
if !isNetworkError(err) {
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
|
||||
incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
|
||||
|
||||
@@ -30,7 +30,7 @@ import (
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
+71
-34
@@ -51,6 +51,7 @@ import (
|
||||
sse "github.com/minio/minio/internal/bucket/encryption"
|
||||
objectlock "github.com/minio/minio/internal/bucket/object/lock"
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/event"
|
||||
@@ -60,9 +61,8 @@ import (
|
||||
"github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
iampolicy "github.com/minio/pkg/iam/policy"
|
||||
"github.com/minio/pkg/sync/errgroup"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
"github.com/minio/pkg/v2/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -367,10 +367,10 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
// Use the following trick to filter in place
|
||||
// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
|
||||
for _, bucketInfo := range bucketsInfo {
|
||||
if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.ListBucketAction,
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucketInfo.Name,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
@@ -379,10 +379,10 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
|
||||
}) {
|
||||
bucketsInfo[n] = bucketInfo
|
||||
n++
|
||||
} else if globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
} else if globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.GetBucketLocationAction,
|
||||
Action: policy.GetBucketLocationAction,
|
||||
BucketName: bucketInfo.Name,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
IsOwner: owner,
|
||||
@@ -475,9 +475,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
deleteObjectsFn := objectAPI.DeleteObjects
|
||||
if api.CacheAPI() != nil {
|
||||
deleteObjectsFn = api.CacheAPI().DeleteObjects
|
||||
}
|
||||
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty
|
||||
if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
|
||||
@@ -487,9 +484,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
|
||||
objectsToDelete := map[ObjectToDelete]int{}
|
||||
getObjectInfoFn := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfoFn = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
var (
|
||||
hasLockEnabled bool
|
||||
@@ -675,6 +669,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
continue
|
||||
}
|
||||
|
||||
defer globalCacheConfig.Delete(bucket, dobj.ObjectName)
|
||||
|
||||
if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
|
||||
// copy so we can re-add null ID.
|
||||
dobj := dobj
|
||||
@@ -717,7 +713,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
|
||||
continue
|
||||
}
|
||||
logger.LogIf(ctx, os.Sweep())
|
||||
os.Sweep()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -770,8 +766,8 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
|
||||
if objectLockEnabled {
|
||||
// Creating a bucket with locking requires the user having more permissions
|
||||
for _, action := range []iampolicy.Action{iampolicy.PutBucketObjectLockConfigurationAction, iampolicy.PutBucketVersioningAction} {
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
for _, action := range []policy.Action{policy.PutBucketObjectLockConfigurationAction, policy.PutBucketVersioningAction} {
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: action,
|
||||
@@ -795,7 +791,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
|
||||
// check if client is attempting to create more buckets, complain about it.
|
||||
if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
|
||||
logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
|
||||
logger.LogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets))
|
||||
}
|
||||
|
||||
opts := MakeBucketOptions{
|
||||
@@ -990,7 +986,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
if fileName == "" {
|
||||
if name != "file" {
|
||||
if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
|
||||
dec := json.NewDecoder(part)
|
||||
|
||||
@@ -1046,11 +1042,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
break
|
||||
}
|
||||
|
||||
if _, ok := formValues["Key"]; !ok {
|
||||
if keyName, ok := formValues["Key"]; !ok {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
} else if fileName == "" && len(keyName) >= 1 {
|
||||
// if we can't get fileName. We use keyName[0] to fileName
|
||||
fileName = keyName[0]
|
||||
}
|
||||
|
||||
if fileName == "" {
|
||||
@@ -1103,10 +1102,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
if len(fanOutEntries) > 0 {
|
||||
// Once signature is validated, check if the user has
|
||||
// explicit permissions for the user.
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.PutObjectFanOutAction,
|
||||
Action: policy.PutObjectFanOutAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
BucketName: bucket,
|
||||
ObjectName: object,
|
||||
@@ -1119,10 +1118,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
} else {
|
||||
// Once signature is validated, check if the user has
|
||||
// explicit permissions for the user.
|
||||
if !globalIAMSys.IsAllowed(iampolicy.Args{
|
||||
if !globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
Groups: cred.Groups,
|
||||
Action: iampolicy.PutObjectAction,
|
||||
Action: policy.PutObjectAction,
|
||||
ConditionValues: getConditionValues(r, "", cred),
|
||||
BucketName: bucket,
|
||||
ObjectName: object,
|
||||
@@ -1140,9 +1139,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize)
|
||||
hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1197,7 +1195,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
})
|
||||
|
||||
var opts ObjectOptions
|
||||
opts, err = putOpts(ctx, r, bucket, object, metadata)
|
||||
opts, err = putOptsFromReq(ctx, r, bucket, object, metadata)
|
||||
if err != nil {
|
||||
writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
|
||||
return
|
||||
@@ -1255,7 +1253,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
// do not try to verify encrypted content/
|
||||
hashReader, err = hash.NewReader(reader, -1, "", "", -1)
|
||||
hashReader, err = hash.NewReader(ctx, reader, -1, "", "", -1)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
@@ -1290,7 +1288,10 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
// instead of "copying" from source, we need the stream to be seekable
|
||||
// to ensure that we can make fan-out calls concurrently.
|
||||
buf := bytebufferpool.Get()
|
||||
defer bytebufferpool.Put(buf)
|
||||
defer func() {
|
||||
buf.Reset()
|
||||
bytebufferpool.Put(buf)
|
||||
}()
|
||||
|
||||
md5w := md5.New()
|
||||
|
||||
@@ -1345,6 +1346,22 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
continue
|
||||
}
|
||||
|
||||
asize, err := objInfo.GetActualSize()
|
||||
if err != nil {
|
||||
asize = objInfo.Size
|
||||
}
|
||||
|
||||
globalCacheConfig.Set(&cache.ObjectInfo{
|
||||
Key: objInfo.Name,
|
||||
Bucket: objInfo.Bucket,
|
||||
ETag: getDecryptedETag(formValues, objInfo, false),
|
||||
ModTime: objInfo.ModTime,
|
||||
Expires: objInfo.Expires.UTC().Format(http.TimeFormat),
|
||||
CacheControl: objInfo.CacheControl,
|
||||
Metadata: cleanReservedKeys(objInfo.UserDefined),
|
||||
Size: asize,
|
||||
})
|
||||
|
||||
fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
|
||||
Key: objInfo.Name,
|
||||
ETag: getDecryptedETag(formValues, objInfo, false),
|
||||
@@ -1401,10 +1418,12 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
return
|
||||
}
|
||||
|
||||
etag := getDecryptedETag(formValues, objInfo, false)
|
||||
|
||||
// We must not use the http.Header().Set method here because some (broken)
|
||||
// clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive).
|
||||
// Therefore, we have to set the ETag directly as map entry.
|
||||
w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
|
||||
w.Header()[xhttp.ETag] = []string{`"` + etag + `"`}
|
||||
|
||||
// Set the relevant version ID as part of the response header.
|
||||
if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
|
||||
@@ -1415,6 +1434,22 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
w.Header().Set(xhttp.Location, obj)
|
||||
}
|
||||
|
||||
asize, err := objInfo.GetActualSize()
|
||||
if err != nil {
|
||||
asize = objInfo.Size
|
||||
}
|
||||
|
||||
defer globalCacheConfig.Set(&cache.ObjectInfo{
|
||||
Key: objInfo.Name,
|
||||
Bucket: objInfo.Bucket,
|
||||
ETag: etag,
|
||||
ModTime: objInfo.ModTime,
|
||||
Expires: objInfo.ExpiresStr(),
|
||||
CacheControl: objInfo.CacheControl,
|
||||
Metadata: cleanReservedKeys(objInfo.UserDefined),
|
||||
Size: asize,
|
||||
})
|
||||
|
||||
// Notify object created event.
|
||||
defer sendEvent(eventArgs{
|
||||
EventName: event.ObjectCreatedPost,
|
||||
@@ -1498,7 +1533,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
// Check if anonymous (non-owner) has access to list objects.
|
||||
readable := globalPolicySys.IsAllowed(policy.Args{
|
||||
readable := globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||
Action: policy.ListBucketAction,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
||||
@@ -1506,7 +1541,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
|
||||
})
|
||||
|
||||
// Check if anonymous (non-owner) has access to upload objects.
|
||||
writable := globalPolicySys.IsAllowed(policy.Args{
|
||||
writable := globalPolicySys.IsAllowed(policy.BucketPolicyArgs{
|
||||
Action: policy.PutObjectAction,
|
||||
BucketName: bucket,
|
||||
ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
|
||||
@@ -1622,9 +1657,11 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
// Return an error if the bucket does not exist
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil && !forceDelete {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
if !forceDelete {
|
||||
if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Attempt to delete bucket.
|
||||
|
||||
@@ -25,6 +25,7 @@ type lcEventSrc uint8
|
||||
//revive:disable:var-naming Underscores is used here to indicate where common prefix ends and the enumeration name begins
|
||||
const (
|
||||
lcEventSrc_None lcEventSrc = iota
|
||||
lcEventSrc_Heal
|
||||
lcEventSrc_Scanner
|
||||
lcEventSrc_Decom
|
||||
lcEventSrc_Rebal
|
||||
|
||||
@@ -22,12 +22,13 @@ import (
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/bucket/lifecycle"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -86,6 +87,41 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
return
|
||||
}
|
||||
|
||||
// Create a map of updated set of rules in request
|
||||
updatedRules := make(map[string]lifecycle.Rule, len(bucketLifecycle.Rules))
|
||||
for _, rule := range bucketLifecycle.Rules {
|
||||
updatedRules[rule.ID] = rule
|
||||
}
|
||||
|
||||
// Get list of rules for the bucket from disk
|
||||
meta, err := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
expiryRuleRemoved := false
|
||||
if len(meta.LifecycleConfigXML) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
for _, rl := range lcCfg.Rules {
|
||||
updRule, ok := updatedRules[rl.ID]
|
||||
// original rule had expiry that is no longer in the new config,
|
||||
// or rule is present but missing expiration flags
|
||||
if (!rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull()) &&
|
||||
(!ok || (updRule.Expiration.IsNull() && updRule.NoncurrentVersionExpiration.IsNull())) {
|
||||
expiryRuleRemoved = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if bucketLifecycle.HasExpiry() || expiryRuleRemoved {
|
||||
currtime := time.Now()
|
||||
bucketLifecycle.ExpiryUpdatedAt = &currtime
|
||||
}
|
||||
|
||||
configData, err := xml.Marshal(bucketLifecycle)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
@@ -142,6 +178,8 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
// explicitly set ExpiryUpdatedAt nil as its meant for internal consumption only
|
||||
config.ExpiryUpdatedAt = nil
|
||||
|
||||
configData, err := xml.Marshal(config)
|
||||
if err != nil {
|
||||
|
||||
+38
-13
@@ -41,8 +41,10 @@ import (
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/s3select"
|
||||
"github.com/minio/pkg/env"
|
||||
"github.com/minio/pkg/workers"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
xnet "github.com/minio/pkg/v2/net"
|
||||
"github.com/minio/pkg/v2/workers"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -217,17 +219,24 @@ type transitionState struct {
|
||||
numWorkers int
|
||||
killCh chan struct{}
|
||||
|
||||
activeTasks int32
|
||||
activeTasks atomic.Int64
|
||||
missedImmediateTasks atomic.Int64
|
||||
|
||||
lastDayMu sync.RWMutex
|
||||
lastDayStats map[string]*lastDayTierStats
|
||||
}
|
||||
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
|
||||
task := transitionTask{objInfo: oi, event: event, src: src}
|
||||
select {
|
||||
case <-t.ctx.Done():
|
||||
case t.transitionCh <- transitionTask{objInfo: oi, event: event, src: src}:
|
||||
case t.transitionCh <- task:
|
||||
default:
|
||||
switch src {
|
||||
case lcEventSrc_s3PutObject, lcEventSrc_s3CopyObject, lcEventSrc_s3CompleteMultipartUpload:
|
||||
// Update missed immediate tasks only for incoming requests.
|
||||
t.missedImmediateTasks.Add(1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,7 +246,7 @@ var globalTransitionState *transitionState
|
||||
// via its Init method.
|
||||
func newTransitionState(ctx context.Context) *transitionState {
|
||||
return &transitionState{
|
||||
transitionCh: make(chan transitionTask, 10000),
|
||||
transitionCh: make(chan transitionTask, 100000),
|
||||
ctx: ctx,
|
||||
killCh: make(chan struct{}),
|
||||
lastDayStats: make(map[string]*lastDayTierStats),
|
||||
@@ -262,8 +271,14 @@ func (t *transitionState) PendingTasks() int {
|
||||
}
|
||||
|
||||
// ActiveTasks returns the number of active (ongoing) ILM transition tasks.
|
||||
func (t *transitionState) ActiveTasks() int {
|
||||
return int(atomic.LoadInt32(&t.activeTasks))
|
||||
func (t *transitionState) ActiveTasks() int64 {
|
||||
return t.activeTasks.Load()
|
||||
}
|
||||
|
||||
// MissedImmediateTasks returns the number of tasks - deferred to scanner due
|
||||
// to tasks channel being backlogged.
|
||||
func (t *transitionState) MissedImmediateTasks() int64 {
|
||||
return t.missedImmediateTasks.Load()
|
||||
}
|
||||
|
||||
// worker waits for transition tasks
|
||||
@@ -278,10 +293,14 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
atomic.AddInt32(&t.activeTasks, 1)
|
||||
t.activeTasks.Add(1)
|
||||
if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
|
||||
logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
|
||||
task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
|
||||
if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !xnet.IsNetworkOrHostDown(err, false) {
|
||||
if !strings.Contains(err.Error(), "use of closed network connection") {
|
||||
logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
|
||||
task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
ts := tierStats{
|
||||
TotalSize: uint64(task.objInfo.Size),
|
||||
@@ -292,7 +311,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
|
||||
}
|
||||
t.addLastDayStats(task.event.StorageClass, ts)
|
||||
}
|
||||
atomic.AddInt32(&t.activeTasks, -1)
|
||||
t.activeTasks.Add(-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -442,7 +461,8 @@ func genTransitionObjName(bucket string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
us := u.String()
|
||||
obj := fmt.Sprintf("%s/%s/%s/%s/%s", globalDeploymentID, bucket, us[0:2], us[2:4], us)
|
||||
hash := xxh3.HashString(pathJoin(globalDeploymentID(), bucket))
|
||||
obj := fmt.Sprintf("%s/%s/%s/%s", strconv.FormatUint(hash, 16), us[0:2], us[2:4], us)
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
@@ -487,9 +507,13 @@ func auditTierActions(ctx context.Context, tier string, bytes int64) func(err er
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
op.TimeToResponseNS = time.Since(startTime).Nanoseconds()
|
||||
since := time.Since(startTime)
|
||||
op.TimeToResponseNS = since.Nanoseconds()
|
||||
globalTierMetrics.Observe(tier, since)
|
||||
globalTierMetrics.logSuccess(tier)
|
||||
} else {
|
||||
op.Error = err.Error()
|
||||
globalTierMetrics.logFailure(tier)
|
||||
}
|
||||
|
||||
logger.GetReqInfo(ctx).AppendTags("tierStats", op)
|
||||
@@ -860,6 +884,7 @@ func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
|
||||
UserTags: oi.UserTags,
|
||||
VersionID: oi.VersionID,
|
||||
ModTime: oi.ModTime,
|
||||
Size: oi.Size,
|
||||
IsLatest: oi.IsLatest,
|
||||
NumVersions: oi.NumVersions,
|
||||
DeleteMarker: oi.DeleteMarker,
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// Validate all the ListObjects query arguments, returns an APIErrorCode
|
||||
@@ -231,7 +231,7 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
|
||||
if token == "" {
|
||||
return token, -1
|
||||
}
|
||||
i := strings.Index(token, "@")
|
||||
i := strings.Index(token, ":")
|
||||
if i < 0 {
|
||||
return token, -1
|
||||
}
|
||||
|
||||
+85
-32
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sync"
|
||||
@@ -35,8 +36,8 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/sync/errgroup"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
"github.com/minio/pkg/v2/sync/errgroup"
|
||||
)
|
||||
|
||||
// BucketMetadataSys captures all bucket metadata for a given cluster.
|
||||
@@ -151,19 +152,66 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
|
||||
return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
|
||||
}
|
||||
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
return updatedAt, err
|
||||
err = sys.save(ctx, meta)
|
||||
return updatedAt, err
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
|
||||
objAPI := newObjectLayerFn()
|
||||
if objAPI == nil {
|
||||
return errServerNotInitialized
|
||||
}
|
||||
|
||||
sys.Set(bucket, meta)
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket) // Do not use caller context here
|
||||
if isMinioMetaBucketName(meta.Name) {
|
||||
return errInvalidArgument
|
||||
}
|
||||
|
||||
return updatedAt, nil
|
||||
if err := meta.Save(ctx, objAPI); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sys.Set(meta.Name, meta)
|
||||
globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
|
||||
return nil
|
||||
}
|
||||
|
||||
// Delete delete the bucket metadata for the specified bucket.
|
||||
// must be used by all callers instead of using Update() with nil configData.
|
||||
func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) {
|
||||
if configFile == bucketLifecycleConfig {
|
||||
// Get bucket config from current site
|
||||
meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket)
|
||||
if e != nil && !errors.Is(e, errConfigNotFound) {
|
||||
return updatedAt, e
|
||||
}
|
||||
var expiryRuleRemoved bool
|
||||
if len(meta.LifecycleConfigXML) > 0 {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
// find a single expiry rule set the flag
|
||||
for _, rl := range lcCfg.Rules {
|
||||
if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() {
|
||||
expiryRuleRemoved = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Form empty ILM details with `ExpiryUpdatedAt` field and save
|
||||
var cfgData []byte
|
||||
if expiryRuleRemoved {
|
||||
var lcCfg lifecycle.Lifecycle
|
||||
currtime := time.Now()
|
||||
lcCfg.ExpiryUpdatedAt = &currtime
|
||||
cfgData, err = xml.Marshal(lcCfg)
|
||||
if err != nil {
|
||||
return updatedAt, err
|
||||
}
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, cfgData, false)
|
||||
}
|
||||
return sys.updateAndParse(ctx, bucket, configFile, nil, false)
|
||||
}
|
||||
|
||||
@@ -254,7 +302,10 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
|
||||
}
|
||||
return nil, time.Time{}, err
|
||||
}
|
||||
if meta.lifecycleConfig == nil {
|
||||
// there could be just `ExpiryUpdatedAt` field populated as part
|
||||
// of last delete all. Treat this situation as not lifecycle configuration
|
||||
// available
|
||||
if meta.lifecycleConfig == nil || len(meta.lifecycleConfig.Rules) == 0 {
|
||||
return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket}
|
||||
}
|
||||
return meta.lifecycleConfig, meta.LifecycleConfigUpdatedAt, nil
|
||||
@@ -297,7 +348,7 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
|
||||
|
||||
// GetPolicyConfig returns configured bucket policy
|
||||
// The returned object may not be modified.
|
||||
func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
|
||||
func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.BucketPolicy, time.Time, error) {
|
||||
meta, _, err := sys.GetConfig(GlobalContext, bucket)
|
||||
if err != nil {
|
||||
if errors.Is(err, errConfigNotFound) {
|
||||
@@ -424,21 +475,8 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) loadBucketMetadata(ctx context.Context, bucket BucketInfo) error {
|
||||
meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket.Name] = meta
|
||||
sys.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// concurrently load bucket metadata to speed up loading bucket metadata.
|
||||
func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo) {
|
||||
func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, failedBuckets map[string]struct{}) {
|
||||
g := errgroup.WithNErrs(len(buckets))
|
||||
bucketMetas := make([]BucketMetadata, len(buckets))
|
||||
for index := range buckets {
|
||||
@@ -478,6 +516,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
|
||||
|
||||
for i, meta := range bucketMetas {
|
||||
if errs[i] != nil {
|
||||
if failedBuckets == nil {
|
||||
failedBuckets = make(map[string]struct{})
|
||||
}
|
||||
failedBuckets[buckets[i].Name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
globalEventNotifier.set(buckets[i], meta) // set notification targets
|
||||
@@ -485,7 +527,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, failedBuckets map[string]struct{}) {
|
||||
const bucketMetadataRefresh = 15 * time.Minute
|
||||
|
||||
sleeper := newDynamicSleeper(2, 150*time.Millisecond, false)
|
||||
@@ -500,7 +542,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
break
|
||||
}
|
||||
|
||||
// Handle if we have some buckets in-memory those are stale.
|
||||
@@ -512,38 +554,49 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
}
|
||||
sys.RemoveStaleBuckets(diskBuckets)
|
||||
|
||||
for _, bucket := range buckets {
|
||||
for i := range buckets {
|
||||
wait := sleeper.Timer(ctx)
|
||||
|
||||
err := sys.loadBucketMetadata(ctx, bucket)
|
||||
meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[i].Name)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
wait() // wait to proceed to next entry.
|
||||
continue
|
||||
}
|
||||
|
||||
sys.Lock()
|
||||
sys.metadataMap[buckets[i].Name] = meta
|
||||
sys.Unlock()
|
||||
|
||||
// Initialize the failed buckets
|
||||
if _, ok := failedBuckets[buckets[i].Name]; ok {
|
||||
globalEventNotifier.set(buckets[i], meta)
|
||||
globalBucketTargetSys.set(buckets[i], meta)
|
||||
delete(failedBuckets, buckets[i].Name)
|
||||
}
|
||||
|
||||
wait() // wait to proceed to next entry.
|
||||
}
|
||||
|
||||
t.Reset(bucketMetadataRefresh)
|
||||
}
|
||||
t.Reset(bucketMetadataRefresh)
|
||||
}
|
||||
}
|
||||
|
||||
// Loads bucket metadata for all buckets into BucketMetadataSys.
|
||||
func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
|
||||
count := 100 // load 100 bucket metadata at a time.
|
||||
failedBuckets := make(map[string]struct{})
|
||||
for {
|
||||
if len(buckets) < count {
|
||||
sys.concurrentLoad(ctx, buckets)
|
||||
sys.concurrentLoad(ctx, buckets, failedBuckets)
|
||||
break
|
||||
}
|
||||
sys.concurrentLoad(ctx, buckets[:count])
|
||||
sys.concurrentLoad(ctx, buckets[:count], failedBuckets)
|
||||
buckets = buckets[count:]
|
||||
}
|
||||
|
||||
if globalIsDistErasure {
|
||||
go sys.refreshBucketsMetadataLoop(ctx)
|
||||
go sys.refreshBucketsMetadataLoop(ctx, failedBuckets)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+13
-14
@@ -41,7 +41,7 @@ import (
|
||||
"github.com/minio/minio/internal/fips"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
@@ -91,7 +91,7 @@ type BucketMetadata struct {
|
||||
LifecycleConfigUpdatedAt time.Time
|
||||
|
||||
// Unexported fields. Must be updated atomically.
|
||||
policyConfig *policy.Policy
|
||||
policyConfig *policy.BucketPolicy
|
||||
notificationConfig *event.Config
|
||||
lifecycleConfig *lifecycle.Lifecycle
|
||||
objectLockConfig *objectlock.Config
|
||||
@@ -143,39 +143,38 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
|
||||
|
||||
// Load - loads the metadata of bucket by name from ObjectLayer api.
|
||||
// If an error is returned the returned metadata will be default initialized.
|
||||
func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
|
||||
func readBucketMetadata(ctx context.Context, api ObjectLayer, name string) (BucketMetadata, error) {
|
||||
if name == "" {
|
||||
logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
|
||||
return errInvalidArgument
|
||||
return BucketMetadata{}, errInvalidArgument
|
||||
}
|
||||
b := newBucketMetadata(name)
|
||||
configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile)
|
||||
data, err := readConfig(ctx, api, configFile)
|
||||
if err != nil {
|
||||
return err
|
||||
return b, err
|
||||
}
|
||||
if len(data) <= 4 {
|
||||
return fmt.Errorf("loadBucketMetadata: no data")
|
||||
return b, fmt.Errorf("loadBucketMetadata: no data")
|
||||
}
|
||||
// Read header
|
||||
switch binary.LittleEndian.Uint16(data[0:2]) {
|
||||
case bucketMetadataFormat:
|
||||
default:
|
||||
return fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
|
||||
return b, fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
|
||||
}
|
||||
switch binary.LittleEndian.Uint16(data[2:4]) {
|
||||
case bucketMetadataVersion:
|
||||
default:
|
||||
return fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
|
||||
return b, fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
|
||||
}
|
||||
// OK, parse data.
|
||||
_, err = b.UnmarshalMsg(data[4:])
|
||||
b.Name = name // in-case parsing failed for some reason, make sure bucket name is not empty.
|
||||
return err
|
||||
return b, err
|
||||
}
|
||||
|
||||
func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
|
||||
b := newBucketMetadata(bucket)
|
||||
err := b.Load(ctx, objectAPI, b.Name)
|
||||
b, err := readBucketMetadata(ctx, objectAPI, bucket)
|
||||
b.Name = bucket // in-case parsing failed for some reason, make sure bucket name is not empty.
|
||||
if err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
return b, err
|
||||
}
|
||||
@@ -218,7 +217,7 @@ func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket strin
|
||||
// The first error encountered is returned.
|
||||
func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) {
|
||||
if len(b.PolicyConfigJSON) != 0 {
|
||||
b.policyConfig, err = policy.ParseConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name)
|
||||
b.policyConfig, err = policy.ParseBucketPolicyConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
|
||||
@@ -28,7 +28,7 @@ import (
|
||||
"github.com/minio/minio/internal/bucket/replication"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// BucketObjectLockSys - map of bucket and retention configuration.
|
||||
|
||||
@@ -27,7 +27,7 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -84,7 +84,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
|
||||
return
|
||||
}
|
||||
|
||||
bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
||||
bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket)
|
||||
if err != nil {
|
||||
writeErrorResponse(ctx, w, APIError{
|
||||
Code: "MalformedPolicy",
|
||||
|
||||
@@ -29,31 +29,31 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/bucket/policy/condition"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
"github.com/minio/pkg/v2/policy/condition"
|
||||
)
|
||||
|
||||
func getAnonReadOnlyBucketPolicy(bucketName string) *policy.Policy {
|
||||
return &policy.Policy{
|
||||
func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
|
||||
return &policy.BucketPolicy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{
|
||||
policy.NewStatement(
|
||||
Statements: []policy.BPStatement{
|
||||
policy.NewBPStatement(
|
||||
"",
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName)),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
|
||||
return &policy.Policy{
|
||||
func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.BucketPolicy {
|
||||
return &policy.BucketPolicy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{
|
||||
policy.NewStatement(
|
||||
Statements: []policy.BPStatement{
|
||||
policy.NewBPStatement(
|
||||
"",
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
@@ -61,34 +61,34 @@ func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.Policy {
|
||||
policy.GetBucketLocationAction,
|
||||
policy.ListBucketMultipartUploadsAction,
|
||||
),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, "")),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName)),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
|
||||
return &policy.Policy{
|
||||
func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.BucketPolicy {
|
||||
return &policy.BucketPolicy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{
|
||||
policy.NewStatement(
|
||||
Statements: []policy.BPStatement{
|
||||
policy.NewBPStatement(
|
||||
"",
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
policy.NewActionSet(policy.GetObjectAction),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName+"/"+prefix)),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
|
||||
return &policy.Policy{
|
||||
func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.BucketPolicy {
|
||||
return &policy.BucketPolicy{
|
||||
Version: policy.DefaultVersion,
|
||||
Statements: []policy.Statement{
|
||||
policy.NewStatement(
|
||||
Statements: []policy.BPStatement{
|
||||
policy.NewBPStatement(
|
||||
"",
|
||||
policy.Allow,
|
||||
policy.NewPrincipal("*"),
|
||||
@@ -98,7 +98,7 @@ func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
|
||||
policy.ListMultipartUploadPartsAction,
|
||||
policy.PutObjectAction,
|
||||
),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName, prefix)),
|
||||
policy.NewResourceSet(policy.NewResource(bucketName+"/"+prefix)),
|
||||
condition.NewFunctions(),
|
||||
),
|
||||
},
|
||||
@@ -493,13 +493,13 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
|
||||
if recV4.Code != testCase.expectedRespStatus {
|
||||
// Verify whether the bucket policy fetched is same as the one inserted.
|
||||
var expectedPolicy *policy.Policy
|
||||
expectedPolicy, err = policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
|
||||
var expectedPolicy *policy.BucketPolicy
|
||||
expectedPolicy, err = policy.ParseBucketPolicyConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error. %v", err)
|
||||
}
|
||||
var gotPolicy *policy.Policy
|
||||
gotPolicy, err = policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
|
||||
var gotPolicy *policy.BucketPolicy
|
||||
gotPolicy, err = policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error. %v", err)
|
||||
}
|
||||
@@ -530,11 +530,11 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
|
||||
}
|
||||
if recV2.Code == http.StatusOK {
|
||||
// Verify whether the bucket policy fetched is same as the one inserted.
|
||||
expectedPolicy, err := policy.ParseConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
|
||||
expectedPolicy, err := policy.ParseBucketPolicyConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error. %v", err)
|
||||
}
|
||||
gotPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
|
||||
gotPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error. %v", err)
|
||||
}
|
||||
|
||||
+12
-8
@@ -32,20 +32,20 @@ import (
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/bucket/policy"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
// PolicySys - policy subsystem.
|
||||
type PolicySys struct{}
|
||||
|
||||
// Get returns stored bucket policy
|
||||
func (sys *PolicySys) Get(bucket string) (*policy.Policy, error) {
|
||||
func (sys *PolicySys) Get(bucket string) (*policy.BucketPolicy, error) {
|
||||
policy, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket)
|
||||
return policy, err
|
||||
}
|
||||
|
||||
// IsAllowed - checks given policy args is allowed to continue the Rest API.
|
||||
func (sys *PolicySys) IsAllowed(args policy.Args) bool {
|
||||
func (sys *PolicySys) IsAllowed(args policy.BucketPolicyArgs) bool {
|
||||
p, err := sys.Get(args.BucketName)
|
||||
if err == nil {
|
||||
return p.IsAllowed(args)
|
||||
@@ -121,7 +121,7 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
"CurrentTime": {currTime.Format(time.RFC3339)},
|
||||
"EpochTime": {strconv.FormatInt(currTime.Unix(), 10)},
|
||||
"SecureTransport": {strconv.FormatBool(r.TLS != nil)},
|
||||
"SourceIp": {handlers.GetSourceIP(r)},
|
||||
"SourceIp": {handlers.GetSourceIPRaw(r)},
|
||||
"UserAgent": {r.UserAgent()},
|
||||
"Referer": {r.Referer()},
|
||||
"principaltype": {principalType},
|
||||
@@ -137,6 +137,10 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
}
|
||||
|
||||
cloneHeader := r.Header.Clone()
|
||||
if v := cloneHeader.Get("x-amz-signature-age"); v != "" {
|
||||
args["signatureAge"] = []string{v}
|
||||
cloneHeader.Del("x-amz-signature-age")
|
||||
}
|
||||
|
||||
if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" {
|
||||
tag, _ := tags.ParseObjectTags(userTags)
|
||||
@@ -236,7 +240,7 @@ func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[s
|
||||
}
|
||||
|
||||
// PolicyToBucketAccessPolicy converts a MinIO policy into a minio-go policy data structure.
|
||||
func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.BucketAccessPolicy, error) {
|
||||
func PolicyToBucketAccessPolicy(bucketPolicy *policy.BucketPolicy) (*miniogopolicy.BucketAccessPolicy, error) {
|
||||
// Return empty BucketAccessPolicy for empty bucket policy.
|
||||
if bucketPolicy == nil {
|
||||
return &miniogopolicy.BucketAccessPolicy{Version: policy.DefaultVersion}, nil
|
||||
@@ -258,15 +262,15 @@ func PolicyToBucketAccessPolicy(bucketPolicy *policy.Policy) (*miniogopolicy.Buc
|
||||
return &policyInfo, nil
|
||||
}
|
||||
|
||||
// BucketAccessPolicyToPolicy - converts minio-go/policy.BucketAccessPolicy to policy.Policy.
|
||||
func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*policy.Policy, error) {
|
||||
// BucketAccessPolicyToPolicy - converts minio-go/policy.BucketAccessPolicy to policy.BucketPolicy.
|
||||
func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*policy.BucketPolicy, error) {
|
||||
data, err := json.Marshal(policyInfo)
|
||||
if err != nil {
|
||||
// This should not happen because policyInfo is valid to convert to JSON data.
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var bucketPolicy policy.Policy
|
||||
var bucketPolicy policy.BucketPolicy
|
||||
json := jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
if err = json.Unmarshal(data, &bucketPolicy); err != nil {
|
||||
// This should not happen because data is valid to JSON data.
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user