mirror of
https://github.com/pgsty/minio.git
synced 2026-10-19 07:56:09 +03:00
Compare commits
132 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f56a182b71 | |||
| 317b40ef90 | |||
| e938ece492 | |||
| 02331a612c | |||
| 8317557f70 | |||
| 1bb7a2a295 | |||
| 215ca58d6a | |||
| 12f570a307 | |||
| e4b619ce1a | |||
| 0a286153bb | |||
| 22d59e757d | |||
| 0daa2dbf59 | |||
| 96c2304ae8 | |||
| 343dd2f491 | |||
| 38f35463b7 | |||
| 5573986e8e | |||
| f3367a1b20 | |||
| a3c2f7b0e8 | |||
| 8fbec30998 | |||
| a7466eeb0e | |||
| 8b1e819bf3 | |||
| fe63664164 | |||
| 4598827dcb | |||
| 9afdb05bf4 | |||
| 9569a85cee | |||
| 54721b7c7b | |||
| 91d8bddbd1 | |||
| 80adc87a14 | |||
| 117ad1b65b | |||
| 2229509362 | |||
| 6ef8e87492 | |||
| 0a25083fdb | |||
| 15137d0327 | |||
| 8c9974bc0f | |||
| 079b6c2b50 | |||
| 0924b34a17 | |||
| 754f7a8a39 | |||
| 64bafe1dfe | |||
| c3e456e7e6 | |||
| 57aaeafd2f | |||
| 3c2e1a87e2 | |||
| da95a2d13f | |||
| cc5e05fdeb | |||
| a79c390cca | |||
| 8a56af439c | |||
| f6e581ce54 | |||
| 8953f88780 | |||
| 4b4a98d5e5 | |||
| 7472818d94 | |||
| ad44fe8d3e | |||
| 55e713db0a | |||
| 33322e6638 | |||
| a1792ca0d1 | |||
| ac8c43fe9c | |||
| 4d40ee00e9 | |||
| 06f59ad631 | |||
| 877e0cac03 | |||
| ef67c39910 | |||
| 508710f4d1 | |||
| 3aa3d9cf14 | |||
| c2fedb4c3f | |||
| 03dc65e12d | |||
| b8d62a8068 | |||
| dbc2368a7b | |||
| 54aed421b8 | |||
| d5e8dac1cf | |||
| 96ec8fcba1 | |||
| 0663eb69ed | |||
| 0594d37230 | |||
| 3cc30bcc18 | |||
| 99c1a642a4 | |||
| c60f54e5be | |||
| 483389f2e2 | |||
| c0f2f84285 | |||
| 069d118329 | |||
| a7b1834772 | |||
| 6415dec37a | |||
| 74253e1ddc | |||
| 01b3fb91e5 | |||
| 2dc917e87f | |||
| 0a284a1a10 | |||
| 5c8339e1e8 | |||
| fd37418da2 | |||
| bbfea29c2b | |||
| aa703dc903 | |||
| 8cd80fec8c | |||
| 780882efcf | |||
| c5636143c6 | |||
| ba6218b354 | |||
| 8e32de3ba9 | |||
| e37508fb8f | |||
| b46a717425 | |||
| 7926df0b80 | |||
| 557df666fd | |||
| f91b257f50 | |||
| 28a2d1eb3d | |||
| a0ae1489e5 | |||
| edfb310a59 | |||
| a2312028b9 | |||
| 78f1f69d57 | |||
| e1e33077e8 | |||
| b3e7de010d | |||
| f5b04865f4 | |||
| bf1c6edb76 | |||
| 2ac7fee017 | |||
| 128256e3ab | |||
| a66a7f3e97 | |||
| 20b79f8945 | |||
| 9a877734b2 | |||
| 409c391850 | |||
| 763ff085a6 | |||
| 5b9656374c | |||
| b32014549c | |||
| 9476d212bc | |||
| 000928d34e | |||
| 6829ae5b13 | |||
| f09756443d | |||
| 5512016885 | |||
| 21ecb941fe | |||
| 77e94087cf | |||
| 9ab1f25a47 | |||
| aaab7aefbe | |||
| 5b8599e52d | |||
| 74e0c9ab9b | |||
| dcce83b288 | |||
| f731e7ea36 | |||
| ec30bb89a4 | |||
| 7cd08594f6 | |||
| 2b4531f069 | |||
| 11544a62aa | |||
| 18550387d5 | |||
| efb03e19e6 |
@@ -1,6 +1,6 @@
|
||||
## Community Contribution License
|
||||
All community contributions in this pull request are licensed to the project maintainers
|
||||
under the terms of the [Apache 2 license] (https://www.apache.org/licenses/LICENSE-2.0).
|
||||
under the terms of the [Apache 2 license](https://www.apache.org/licenses/LICENSE-2.0).
|
||||
By creating this pull request I represent that I have the right to license the
|
||||
contributions to the project maintainers under the Apache 2 license.
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -16,10 +16,10 @@ x-minio-common: &minio-common
|
||||
MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
|
||||
MINIO_ROOT_PASSWORD: "minio123"
|
||||
MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
test: ["CMD", "mc", "ready", "local"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
@@ -4,6 +4,7 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
# This ensures that previous jobs for the PR are canceled when the PR is
|
||||
# updated.
|
||||
|
||||
@@ -3,9 +3,11 @@ on:
|
||||
pull_request:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
push:
|
||||
branches:
|
||||
- master
|
||||
- next
|
||||
|
||||
permissions:
|
||||
contents: read # to fetch code (actions/checkout)
|
||||
@@ -20,7 +22,7 @@ jobs:
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v3
|
||||
with:
|
||||
go-version: 1.21.1
|
||||
go-version: 1.21.4
|
||||
check-latest: true
|
||||
- name: Get official govulncheck
|
||||
run: go install golang.org/x/vuln/cmd/govulncheck@latest
|
||||
|
||||
@@ -42,3 +42,4 @@ docs/debugging/healing-bin/healing-bin
|
||||
docs/debugging/inspect/inspect
|
||||
docs/debugging/pprofgoparser/pprofgoparser
|
||||
docs/debugging/reorder-disks/reorder-disks
|
||||
docs/debugging/populate-hard-links/populate-hardlinks
|
||||
|
||||
@@ -2416,6 +2416,60 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/VividCortex/ewma
|
||||
https://github.com/VividCortex/ewma
|
||||
----------------------------------------------------------------
|
||||
The MIT License
|
||||
|
||||
Copyright (c) 2013 VividCortex
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/acarl005/stripansi
|
||||
https://github.com/acarl005/stripansi
|
||||
----------------------------------------------------------------
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2018 Andrew Carlson
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/alecthomas/participle
|
||||
https://github.com/alecthomas/participle
|
||||
----------------------------------------------------------------
|
||||
@@ -4495,34 +4549,6 @@ WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
|
||||
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
|
||||
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/djherbis/atime
|
||||
https://github.com/djherbis/atime
|
||||
----------------------------------------------------------------
|
||||
The MIT License (MIT)
|
||||
|
||||
Copyright (c) 2015 Dustin H
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/docker/go-units
|
||||
@@ -15244,6 +15270,213 @@ https://github.com/matttproud/golang_protobuf_extensions
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/matttproud/golang_protobuf_extensions/v2
|
||||
https://github.com/matttproud/golang_protobuf_extensions/v2
|
||||
----------------------------------------------------------------
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/miekg/dns
|
||||
https://github.com/miekg/dns
|
||||
----------------------------------------------------------------
|
||||
@@ -25833,6 +26066,214 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
||||
THE SOFTWARE.
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/safchain/ethtool
|
||||
https://github.com/safchain/ethtool
|
||||
----------------------------------------------------------------
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "{}"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright {yyyy} {name of copyright owner}
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/secure-io/sio-go
|
||||
@@ -27106,6 +27547,36 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/vbauerster/mpb/v8
|
||||
https://github.com/vbauerster/mpb/v8
|
||||
----------------------------------------------------------------
|
||||
This is free and unencumbered software released into the public domain.
|
||||
|
||||
Anyone is free to copy, modify, publish, use, compile, sell, or
|
||||
distribute this software, either in source code form or as a compiled
|
||||
binary, for any purpose, commercial or non-commercial, and by any
|
||||
means.
|
||||
|
||||
In jurisdictions that recognize copyright laws, the author or authors
|
||||
of this software dedicate any and all copyright interest in the
|
||||
software to the public domain. We make this dedication for the benefit
|
||||
of the public at large and to the detriment of our heirs and
|
||||
successors. We intend this dedication to be an overt act of
|
||||
relinquishment in perpetuity of all present and future rights to this
|
||||
software under copyright law.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
||||
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
||||
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
||||
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
|
||||
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
|
||||
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
|
||||
OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
For more information, please refer to <http://unlicense.org/>
|
||||
|
||||
================================================================
|
||||
|
||||
github.com/xdg/scram
|
||||
|
||||
+1
-3
@@ -1,8 +1,6 @@
|
||||
FROM minio/minio:latest
|
||||
|
||||
ENV PATH=/opt/bin:$PATH
|
||||
|
||||
COPY ./minio /opt/bin/minio
|
||||
COPY ./minio /usr/bin/minio
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
+35
-23
@@ -1,4 +1,31 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.0
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:9.2
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
@@ -17,33 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
PATH=/opt/bin:$PATH
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||
|
||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
|
||||
RUN \
|
||||
microdnf clean all && \
|
||||
microdnf update --nodocs && \
|
||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
|
||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
|
||||
curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
|
||||
microdnf clean all && \
|
||||
chmod +x /opt/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/verify-minio.sh && \
|
||||
/usr/bin/verify-minio.sh && \
|
||||
microdnf clean all
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
|
||||
+33
-25
@@ -1,6 +1,31 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.0
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
@@ -19,35 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
PATH=/opt/bin:$PATH
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||
|
||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
|
||||
RUN \
|
||||
microdnf clean all && \
|
||||
microdnf update --nodocs && \
|
||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
|
||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
|
||||
microdnf clean all && \
|
||||
chmod +x /opt/bin/minio && \
|
||||
chmod +x /opt/bin/mc && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/verify-minio.sh && \
|
||||
/usr/bin/verify-minio.sh && \
|
||||
microdnf clean all
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
|
||||
+26
-24
@@ -1,6 +1,25 @@
|
||||
FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.0
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi9/ubi-micro:9.2
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
@@ -18,34 +37,17 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
PATH=/opt/bin:$PATH
|
||||
MINIO_CONFIG_ENV_FILE=config.env
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
|
||||
COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
|
||||
RUN \
|
||||
microdnf clean all && \
|
||||
microdnf update --nodocs && \
|
||||
rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
|
||||
microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
|
||||
mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
|
||||
microdnf clean all && \
|
||||
chmod +x /opt/bin/minio && \
|
||||
chmod +x /usr/bin/docker-entrypoint.sh && \
|
||||
chmod +x /usr/bin/verify-minio.sh && \
|
||||
/usr/bin/verify-minio.sh && \
|
||||
microdnf clean all
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
FROM golang:1.21-alpine as build
|
||||
|
||||
ARG TARGETARCH
|
||||
ARG RELEASE
|
||||
|
||||
ENV GOPATH /go
|
||||
ENV CGO_ENABLED 0
|
||||
|
||||
# Install curl and minisign
|
||||
RUN apk add -U --no-cache ca-certificates && \
|
||||
apk add -U --no-cache curl && \
|
||||
go install aead.dev/minisign/cmd/minisign@v0.2.0
|
||||
|
||||
# Download minio binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
|
||||
curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
|
||||
chmod +x /go/bin/minio
|
||||
|
||||
# Download mc binary and signature file
|
||||
RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
|
||||
curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
|
||||
chmod +x /go/bin/mc
|
||||
|
||||
# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
|
||||
RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
|
||||
minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
|
||||
|
||||
FROM registry.access.redhat.com/ubi8/ubi-micro:latest
|
||||
|
||||
ARG RELEASE
|
||||
|
||||
LABEL name="MinIO" \
|
||||
vendor="MinIO Inc <dev@min.io>" \
|
||||
maintainer="MinIO Inc <dev@min.io>" \
|
||||
version="${RELEASE}" \
|
||||
release="${RELEASE}" \
|
||||
summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
|
||||
description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
|
||||
|
||||
ENV MINIO_ACCESS_KEY_FILE=access_key \
|
||||
MINIO_SECRET_KEY_FILE=secret_key \
|
||||
MINIO_ROOT_USER_FILE=access_key \
|
||||
MINIO_ROOT_PASSWORD_FILE=secret_key \
|
||||
MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
|
||||
MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
|
||||
MINIO_CONFIG_ENV_FILE=config.env \
|
||||
MC_CONFIG_DIR=/tmp/.mc
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /go/bin/minio /usr/bin/minio
|
||||
COPY --from=build /go/bin/mc /usr/bin/mc
|
||||
|
||||
COPY CREDITS /licenses/CREDITS
|
||||
COPY LICENSE /licenses/LICENSE
|
||||
COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
|
||||
|
||||
EXPOSE 9000
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
|
||||
CMD ["minio"]
|
||||
@@ -87,8 +87,8 @@ function verify_rewrite() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
go build ./docs/debugging/s3-check-md5/
|
||||
if ! ./s3-check-md5 \
|
||||
go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key minio \
|
||||
@@ -113,7 +113,7 @@ function verify_rewrite() {
|
||||
go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
|
||||
sleep 1
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-versions \
|
||||
-access-key minio \
|
||||
|
||||
@@ -82,8 +82,8 @@ function start_minio_16drive() {
|
||||
rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
|
||||
sleep 10
|
||||
|
||||
go build ./docs/debugging/s3-check-md5/
|
||||
if ! ./s3-check-md5 \
|
||||
go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
@@ -111,7 +111,7 @@ function start_minio_16drive() {
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
@@ -134,7 +134,7 @@ function start_minio_16drive() {
|
||||
|
||||
"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
|
||||
|
||||
if ! ./s3-check-md5 \
|
||||
if ! s3-check-md5 \
|
||||
-debug \
|
||||
-access-key minio \
|
||||
-secret-key minio123 \
|
||||
|
||||
@@ -9,11 +9,6 @@ x-minio-common: &minio-common
|
||||
expose:
|
||||
- "9000"
|
||||
- "9001"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
|
||||
interval: 30s
|
||||
timeout: 20s
|
||||
retries: 3
|
||||
|
||||
# starts 4 docker containers running minio server instances.
|
||||
# using nginx reverse proxy, load balancing, you can access
|
||||
|
||||
@@ -392,7 +392,8 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
// of bucket metadata
|
||||
zipWriter := zip.NewWriter(w)
|
||||
defer zipWriter.Close()
|
||||
rawDataFn := func(r io.Reader, filename string, sz int) error {
|
||||
|
||||
rawDataFn := func(r io.Reader, filename string, sz int) {
|
||||
header, zerr := zip.FileInfoHeader(dummyFileInfo{
|
||||
name: filename,
|
||||
size: int64(sz),
|
||||
@@ -401,20 +402,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
isDir: false,
|
||||
sys: nil,
|
||||
})
|
||||
if zerr != nil {
|
||||
logger.LogIf(ctx, zerr)
|
||||
return nil
|
||||
if zerr == nil {
|
||||
header.Method = zip.Deflate
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
if zerr == nil {
|
||||
io.Copy(zwriter, r)
|
||||
}
|
||||
}
|
||||
header.Method = zip.Deflate
|
||||
zwriter, zerr := zipWriter.CreateHeader(header)
|
||||
if zerr != nil {
|
||||
logger.LogIf(ctx, zerr)
|
||||
return nil
|
||||
}
|
||||
if _, err := io.Copy(zwriter, r); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
cfgFiles := []string{
|
||||
@@ -446,10 +440,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketLifecycleConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -465,10 +456,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketQuotaConfigFile:
|
||||
config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
|
||||
if err != nil {
|
||||
@@ -483,10 +471,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketSSEConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -501,10 +486,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketTaggingConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -519,10 +501,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case objectLockConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -538,10 +517,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketVersioningConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -557,10 +533,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketReplicationConfig:
|
||||
config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
|
||||
if err != nil {
|
||||
@@ -575,11 +548,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
case bucketTargetsFile:
|
||||
config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
|
||||
if err != nil {
|
||||
@@ -595,10 +564,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
|
||||
writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
|
||||
return
|
||||
}
|
||||
rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,7 +28,6 @@ import (
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
"github.com/minio/minio/internal/config/etcd"
|
||||
xldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
@@ -500,8 +499,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
|
||||
switch hkv.Key {
|
||||
case config.EtcdSubSys:
|
||||
off = !etcd.Enabled(item.Config)
|
||||
case config.CacheSubSys:
|
||||
off = !cache.Enabled(item.Config)
|
||||
case config.StorageClassSubSys:
|
||||
off = !storageclass.Enabled(item.Config)
|
||||
case config.PolicyPluginSubSys:
|
||||
|
||||
@@ -121,7 +121,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
|
||||
|
||||
// IDP config is not dynamic. Sanity check.
|
||||
if dynamic {
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
|
||||
writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), "", r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ import (
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/madmin-go/v3"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
@@ -537,7 +538,7 @@ func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.
|
||||
|
||||
connectTime := time.Now()
|
||||
for {
|
||||
n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
|
||||
n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
|
||||
atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
|
||||
|
||||
@@ -781,6 +781,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if sp.Version == "" && len(sp.Statements) == 0 {
|
||||
sp = nil
|
||||
}
|
||||
}
|
||||
|
||||
opts.sessionPolicy = sp
|
||||
@@ -911,6 +914,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if sp.Version == "" && len(sp.Statements) == 0 {
|
||||
sp = nil
|
||||
}
|
||||
}
|
||||
opts := updateServiceAccountOpts{
|
||||
secretKey: updateReq.NewSecretKey,
|
||||
@@ -996,9 +1002,12 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
}
|
||||
}
|
||||
|
||||
// if session policy is nil or empty, then it is implied policy
|
||||
impliedPolicy := sessionPolicy == nil || (sessionPolicy.Version == "" && len(sessionPolicy.Statements) == 0)
|
||||
|
||||
var svcAccountPolicy policy.Policy
|
||||
|
||||
if sessionPolicy != nil {
|
||||
if !impliedPolicy {
|
||||
svcAccountPolicy = *sessionPolicy
|
||||
} else {
|
||||
policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, false)
|
||||
@@ -1025,7 +1034,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
|
||||
Name: svcAccount.Name,
|
||||
Description: svcAccount.Description,
|
||||
AccountStatus: svcAccount.Status,
|
||||
ImpliedPolicy: sessionPolicy == nil,
|
||||
ImpliedPolicy: impliedPolicy,
|
||||
Policy: string(policyJSON),
|
||||
Expiration: expiration,
|
||||
}
|
||||
@@ -1147,7 +1156,11 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
|
||||
|
||||
// We do not care if service account is readable or not at this point,
|
||||
// since this is a delete call we shall allow it to be deleted if possible.
|
||||
svcAccount, _, _ := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
|
||||
svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
|
||||
if errors.Is(err, errNoSuchServiceAccount) {
|
||||
writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
|
||||
AccountName: cred.AccessKey,
|
||||
|
||||
+44
-11
@@ -52,6 +52,7 @@ import (
|
||||
"github.com/minio/minio/internal/dsync"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/mux"
|
||||
@@ -64,6 +65,8 @@ import (
|
||||
const (
|
||||
maxEConfigJSONSize = 262272
|
||||
kubernetesVersionEndpoint = "https://kubernetes.default.svc/version"
|
||||
anonymizeParam = "anonymize"
|
||||
anonymizeStrict = "strict"
|
||||
)
|
||||
|
||||
// Only valid query params for mgmt admin APIs.
|
||||
@@ -754,11 +757,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
|
||||
return
|
||||
}
|
||||
}
|
||||
// read request body
|
||||
io.CopyN(io.Discard, r.Body, 1)
|
||||
|
||||
globalProfilerMu.Lock()
|
||||
|
||||
if globalProfiler == nil {
|
||||
globalProfiler = make(map[string]minioProfiler, 10)
|
||||
}
|
||||
@@ -1001,7 +1001,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
Message: hr.errBody,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
HostID: globalDeploymentID(),
|
||||
})
|
||||
}
|
||||
if !started {
|
||||
@@ -1056,7 +1056,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
respCh := make(chan healResp)
|
||||
respCh := make(chan healResp, 1)
|
||||
switch {
|
||||
case hip.forceStop:
|
||||
go func() {
|
||||
@@ -1218,7 +1218,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
|
||||
totalRx := int64(0)
|
||||
connectTime := time.Now()
|
||||
for {
|
||||
n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
|
||||
n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
|
||||
if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
|
||||
// would mean the network is not stable. Logging here will help in debugging network issues.
|
||||
if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
|
||||
@@ -1400,13 +1400,21 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
|
||||
}
|
||||
|
||||
func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
|
||||
if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
|
||||
if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{VersioningEnabled: globalSiteReplicationSys.isEnabled()}); err != nil {
|
||||
if _, ok := err.(BucketExists); !ok {
|
||||
// Only BucketExists error can be ignored.
|
||||
return false, err
|
||||
}
|
||||
bucketExists = true
|
||||
}
|
||||
|
||||
if globalSiteReplicationSys.isEnabled() {
|
||||
configData := []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>speedtest/*</Prefix></ExcludedPrefixes></VersioningConfiguration>`)
|
||||
if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketVersioningConfig, configData); err != nil {
|
||||
return false, err
|
||||
}
|
||||
}
|
||||
|
||||
return bucketExists, nil
|
||||
}
|
||||
|
||||
@@ -1982,7 +1990,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
|
||||
Domain: domain,
|
||||
Region: globalSite.Region,
|
||||
SQSARN: globalEventNotifier.GetARNList(false),
|
||||
DeploymentID: globalDeploymentID,
|
||||
DeploymentID: globalDeploymentID(),
|
||||
Buckets: buckets,
|
||||
Objects: objects,
|
||||
Versions: versions,
|
||||
@@ -2027,8 +2035,14 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
|
||||
|
||||
func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) {
|
||||
hostAnonymizer := createHostAnonymizer()
|
||||
// anonAddr - Anonymizes hosts in given input string.
|
||||
|
||||
anonParam := query.Get(anonymizeParam)
|
||||
// anonAddr - Anonymizes hosts in given input string
|
||||
// (only if the anonymize param is set to srict).
|
||||
anonAddr := func(addr string) string {
|
||||
if anonParam != anonymizeStrict {
|
||||
return addr
|
||||
}
|
||||
newAddr, found := hostAnonymizer[addr]
|
||||
if found {
|
||||
return newAddr
|
||||
@@ -2089,6 +2103,20 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteNetInfo := func() {
|
||||
if query.Get(string(madmin.HealthDataTypeSysNet)) == "true" {
|
||||
localNetInfo := madmin.GetNetInfo(globalLocalNodeName, globalInternodeInterface)
|
||||
healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, localNetInfo)
|
||||
|
||||
peerNetInfos := globalNotificationSys.GetNetInfo(healthCtx)
|
||||
for _, n := range peerNetInfos {
|
||||
anonymizeAddr(&n)
|
||||
healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, n)
|
||||
}
|
||||
partialWrite(healthInfo)
|
||||
}
|
||||
}
|
||||
|
||||
getAndWriteOSInfo := func() {
|
||||
if query.Get("sysosinfo") == "true" {
|
||||
localOSInfo := madmin.GetOSInfo(healthCtx, globalLocalNodeName)
|
||||
@@ -2189,6 +2217,10 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
}
|
||||
|
||||
anonymizeCmdLine := func(cmdLine string) string {
|
||||
if anonParam != anonymizeStrict {
|
||||
return cmdLine
|
||||
}
|
||||
|
||||
if !globalIsDistErasure {
|
||||
// FS mode - single server - hard code to `server1`
|
||||
anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
|
||||
@@ -2310,6 +2342,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
|
||||
getAndWritePlatformInfo()
|
||||
getAndWriteCPUs()
|
||||
getAndWritePartitions()
|
||||
getAndWriteNetInfo()
|
||||
getAndWriteOSInfo()
|
||||
getAndWriteMemInfo()
|
||||
getAndWriteProcInfo()
|
||||
@@ -2392,7 +2425,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
DeploymentID: globalDeploymentID(),
|
||||
},
|
||||
},
|
||||
}
|
||||
@@ -2690,7 +2723,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
|
||||
ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount
|
||||
ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount
|
||||
|
||||
ci.DeploymentID = globalDeploymentID
|
||||
ci.DeploymentID = globalDeploymentID()
|
||||
ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)
|
||||
|
||||
select {
|
||||
|
||||
@@ -310,6 +310,7 @@ const (
|
||||
ErrSiteReplicationBucketMetaError
|
||||
ErrSiteReplicationIAMError
|
||||
ErrSiteReplicationConfigMissing
|
||||
ErrSiteReplicationIAMConfigMismatch
|
||||
|
||||
// Pool rebalance errors
|
||||
ErrAdminRebalanceAlreadyStarted
|
||||
@@ -1512,6 +1513,11 @@ var errorCodes = errorCodeMap{
|
||||
Description: "Site not found in site replication configuration",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrSiteReplicationIAMConfigMismatch: {
|
||||
Code: "XMinioSiteReplicationIAMConfigMismatch",
|
||||
Description: "IAM configuration mismatch between sites",
|
||||
HTTPStatusCode: http.StatusBadRequest,
|
||||
},
|
||||
ErrAdminRebalanceAlreadyStarted: {
|
||||
Code: "XMinioAdminRebalanceAlreadyStarted",
|
||||
Description: "Pool rebalance is already started",
|
||||
@@ -2337,6 +2343,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
|
||||
case dns.ErrBucketConflict:
|
||||
apiErr = ErrBucketAlreadyExists
|
||||
default:
|
||||
if _, ok := err.(tags.Error); ok {
|
||||
// tag errors are not exported, so we check their custom interface to avoid logging.
|
||||
// The correct type is inserted by toAPIError.
|
||||
apiErr = ErrInternalError
|
||||
break
|
||||
}
|
||||
var ie, iw int
|
||||
// This work-around is to handle the issue golang/go#30648
|
||||
//nolint:gocritic
|
||||
|
||||
@@ -133,11 +133,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
|
||||
w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
|
||||
}
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
|
||||
w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
|
||||
}
|
||||
|
||||
// Set tag count if object has tags
|
||||
if len(objInfo.UserTags) > 0 {
|
||||
tags, _ := tags.ParseObjectTags(objInfo.UserTags)
|
||||
|
||||
+3
-1
@@ -561,6 +561,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
|
||||
}
|
||||
content.UserMetadata.Set(k, v)
|
||||
}
|
||||
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||
content.Internal = &ObjectInternalInfo{
|
||||
K: object.DataBlocks,
|
||||
M: object.ParityBlocks,
|
||||
@@ -704,6 +705,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
|
||||
}
|
||||
content.UserMetadata.Set(k, v)
|
||||
}
|
||||
content.UserMetadata.Set("expires", object.Expires.Format(http.TimeFormat))
|
||||
content.Internal = &ObjectInternalInfo{
|
||||
K: object.DataBlocks,
|
||||
M: object.ParityBlocks,
|
||||
@@ -978,7 +980,7 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
|
||||
BucketName: reqInfo.BucketName,
|
||||
Key: reqInfo.ObjectName,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
HostID: globalDeploymentID(),
|
||||
}
|
||||
encodedErrorResponse := encodeResponseJSON(errorResponse)
|
||||
writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
|
||||
|
||||
@@ -61,18 +61,6 @@ func newObjectLayerFn() ObjectLayer {
|
||||
return globalObjectAPI
|
||||
}
|
||||
|
||||
func newCachedObjectLayerFn() CacheObjectLayer {
|
||||
globalObjLayerMutex.RLock()
|
||||
defer globalObjLayerMutex.RUnlock()
|
||||
return globalCacheObjectAPI
|
||||
}
|
||||
|
||||
func setCacheObjectLayer(c CacheObjectLayer) {
|
||||
globalObjLayerMutex.Lock()
|
||||
globalCacheObjectAPI = c
|
||||
globalObjLayerMutex.Unlock()
|
||||
}
|
||||
|
||||
func setObjectLayer(o ObjectLayer) {
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = o
|
||||
@@ -82,7 +70,6 @@ func setObjectLayer(o ObjectLayer) {
|
||||
// objectAPIHandler implements and provides http handlers for S3 API.
|
||||
type objectAPIHandlers struct {
|
||||
ObjectAPI func() ObjectLayer
|
||||
CacheAPI func() CacheObjectLayer
|
||||
}
|
||||
|
||||
// getHost tries its best to return the request host.
|
||||
@@ -189,7 +176,6 @@ func registerAPIRouter(router *mux.Router) {
|
||||
// Initialize API.
|
||||
api := objectAPIHandlers{
|
||||
ObjectAPI: newObjectLayerFn,
|
||||
CacheAPI: newCachedObjectLayerFn,
|
||||
}
|
||||
|
||||
// API Router
|
||||
|
||||
+111
-110
File diff suppressed because one or more lines are too long
+12
-2
@@ -163,7 +163,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
|
||||
}
|
||||
rd, objInfo, _, err := core.GetObject(ctx, srcBucket, srcObject, gopts)
|
||||
if err != nil {
|
||||
return err
|
||||
return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID)
|
||||
}
|
||||
defer rd.Close()
|
||||
|
||||
@@ -226,7 +226,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
|
||||
}
|
||||
rd, objInfo, _, err := c.GetObject(ctx, srcBucket, srcObject, gopts)
|
||||
if err != nil {
|
||||
return err
|
||||
return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID)
|
||||
}
|
||||
defer rd.Close()
|
||||
|
||||
@@ -262,6 +262,9 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
|
||||
if err := ri.load(ctx, api, job); err != nil {
|
||||
return err
|
||||
}
|
||||
if ri.Complete {
|
||||
return nil
|
||||
}
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
|
||||
delay := job.Replicate.Flags.Retry.Delay
|
||||
@@ -517,6 +520,10 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
|
||||
oi.UserDefined[xhttp.AmzStorageClass] = objInfo.StorageClass
|
||||
}
|
||||
|
||||
for k, v := range objInfo.UserMetadata {
|
||||
oi.UserDefined[k] = v
|
||||
}
|
||||
|
||||
return oi
|
||||
}
|
||||
|
||||
@@ -837,6 +844,9 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
if err := ri.load(ctx, api, job); err != nil {
|
||||
return err
|
||||
}
|
||||
if ri.Complete {
|
||||
return nil
|
||||
}
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
lastObject := ri.Object
|
||||
|
||||
|
||||
@@ -261,6 +261,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
|
||||
if err := ri.load(ctx, api, job); err != nil {
|
||||
return err
|
||||
}
|
||||
if ri.Complete {
|
||||
return nil
|
||||
}
|
||||
|
||||
globalBatchJobsMetrics.save(job.ID, ri)
|
||||
lastObject := ri.Object
|
||||
|
||||
+7
-11
@@ -474,9 +474,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
}
|
||||
|
||||
deleteObjectsFn := objectAPI.DeleteObjects
|
||||
if api.CacheAPI() != nil {
|
||||
deleteObjectsFn = api.CacheAPI().DeleteObjects
|
||||
}
|
||||
|
||||
// Return Malformed XML as S3 spec if the number of objects is empty
|
||||
if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
|
||||
@@ -486,9 +483,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
|
||||
objectsToDelete := map[ObjectToDelete]int{}
|
||||
getObjectInfoFn := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfoFn = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
var (
|
||||
hasLockEnabled bool
|
||||
@@ -716,7 +710,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
|
||||
if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
|
||||
continue
|
||||
}
|
||||
logger.LogIf(ctx, os.Sweep())
|
||||
os.Sweep()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -794,7 +788,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
|
||||
|
||||
// check if client is attempting to create more buckets, complain about it.
|
||||
if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
|
||||
logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
|
||||
logger.LogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets))
|
||||
}
|
||||
|
||||
opts := MakeBucketOptions{
|
||||
@@ -989,7 +983,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
}
|
||||
|
||||
var b bytes.Buffer
|
||||
if fileName == "" {
|
||||
if name != "file" {
|
||||
if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
|
||||
dec := json.NewDecoder(part)
|
||||
|
||||
@@ -1045,11 +1039,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
break
|
||||
}
|
||||
|
||||
if _, ok := formValues["Key"]; !ok {
|
||||
if keyName, ok := formValues["Key"]; !ok {
|
||||
apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
|
||||
apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
|
||||
writeErrorResponse(ctx, w, apiErr, r.URL)
|
||||
return
|
||||
} else if fileName == "" && len(keyName) >= 1 {
|
||||
// if we can't get fileName. We use keyName[0] to fileName
|
||||
fileName = keyName[0]
|
||||
}
|
||||
|
||||
if fileName == "" {
|
||||
@@ -1141,7 +1138,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
|
||||
|
||||
hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
|
||||
+18
-8
@@ -43,6 +43,7 @@ import (
|
||||
"github.com/minio/minio/internal/s3select"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
"github.com/minio/pkg/v2/workers"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -223,11 +224,19 @@ type transitionState struct {
|
||||
lastDayStats map[string]*lastDayTierStats
|
||||
}
|
||||
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
|
||||
select {
|
||||
case <-t.ctx.Done():
|
||||
case t.transitionCh <- transitionTask{objInfo: oi, event: event, src: src}:
|
||||
default:
|
||||
func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc, blocking bool) {
|
||||
task := transitionTask{objInfo: oi, event: event, src: src}
|
||||
if blocking {
|
||||
select {
|
||||
case <-t.ctx.Done():
|
||||
case t.transitionCh <- task:
|
||||
}
|
||||
} else {
|
||||
select {
|
||||
case <-t.ctx.Done():
|
||||
case t.transitionCh <- task:
|
||||
default:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,7 +246,7 @@ var globalTransitionState *transitionState
|
||||
// via its Init method.
|
||||
func newTransitionState(ctx context.Context) *transitionState {
|
||||
return &transitionState{
|
||||
transitionCh: make(chan transitionTask, 10000),
|
||||
transitionCh: make(chan transitionTask, 100000),
|
||||
ctx: ctx,
|
||||
killCh: make(chan struct{}),
|
||||
lastDayStats: make(map[string]*lastDayTierStats),
|
||||
@@ -367,7 +376,7 @@ func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
|
||||
if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
|
||||
switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action {
|
||||
case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
|
||||
globalTransitionState.queueTransitionTask(obj, event, src)
|
||||
globalTransitionState.queueTransitionTask(obj, event, src, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -444,7 +453,8 @@ func genTransitionObjName(bucket string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
us := u.String()
|
||||
obj := fmt.Sprintf("%s/%s/%s/%s/%s", globalDeploymentID, bucket, us[0:2], us[2:4], us)
|
||||
hash := xxh3.HashString(pathJoin(globalDeploymentID(), bucket))
|
||||
obj := fmt.Sprintf("%s/%s/%s/%s", strconv.FormatUint(hash, 16), us[0:2], us[2:4], us)
|
||||
return obj, nil
|
||||
}
|
||||
|
||||
|
||||
+25
-23
@@ -437,21 +437,8 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) loadBucketMetadata(ctx context.Context, bucket BucketInfo) error {
|
||||
meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sys.Lock()
|
||||
sys.metadataMap[bucket.Name] = meta
|
||||
sys.Unlock()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// concurrently load bucket metadata to speed up loading bucket metadata.
|
||||
func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo) {
|
||||
func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, failedBuckets map[string]struct{}) {
|
||||
g := errgroup.WithNErrs(len(buckets))
|
||||
bucketMetas := make([]BucketMetadata, len(buckets))
|
||||
for index := range buckets {
|
||||
@@ -491,6 +478,10 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
|
||||
|
||||
for i, meta := range bucketMetas {
|
||||
if errs[i] != nil {
|
||||
if failedBuckets == nil {
|
||||
failedBuckets = make(map[string]struct{})
|
||||
}
|
||||
failedBuckets[buckets[i].Name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
globalEventNotifier.set(buckets[i], meta) // set notification targets
|
||||
@@ -498,7 +489,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, failedBuckets map[string]struct{}) {
|
||||
const bucketMetadataRefresh = 15 * time.Minute
|
||||
|
||||
sleeper := newDynamicSleeper(2, 150*time.Millisecond, false)
|
||||
@@ -513,7 +504,7 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
continue
|
||||
break
|
||||
}
|
||||
|
||||
// Handle if we have some buckets in-memory those are stale.
|
||||
@@ -525,38 +516,49 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
|
||||
}
|
||||
sys.RemoveStaleBuckets(diskBuckets)
|
||||
|
||||
for _, bucket := range buckets {
|
||||
for i := range buckets {
|
||||
wait := sleeper.Timer(ctx)
|
||||
|
||||
err := sys.loadBucketMetadata(ctx, bucket)
|
||||
meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[i].Name)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
wait() // wait to proceed to next entry.
|
||||
continue
|
||||
}
|
||||
|
||||
sys.Lock()
|
||||
sys.metadataMap[buckets[i].Name] = meta
|
||||
sys.Unlock()
|
||||
|
||||
// Initialize the failed buckets
|
||||
if _, ok := failedBuckets[buckets[i].Name]; ok {
|
||||
globalEventNotifier.set(buckets[i], meta)
|
||||
globalBucketTargetSys.set(buckets[i], meta)
|
||||
delete(failedBuckets, buckets[i].Name)
|
||||
}
|
||||
|
||||
wait() // wait to proceed to next entry.
|
||||
}
|
||||
|
||||
t.Reset(bucketMetadataRefresh)
|
||||
}
|
||||
t.Reset(bucketMetadataRefresh)
|
||||
}
|
||||
}
|
||||
|
||||
// Loads bucket metadata for all buckets into BucketMetadataSys.
|
||||
func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
|
||||
count := 100 // load 100 bucket metadata at a time.
|
||||
failedBuckets := make(map[string]struct{})
|
||||
for {
|
||||
if len(buckets) < count {
|
||||
sys.concurrentLoad(ctx, buckets)
|
||||
sys.concurrentLoad(ctx, buckets, failedBuckets)
|
||||
break
|
||||
}
|
||||
sys.concurrentLoad(ctx, buckets[:count])
|
||||
sys.concurrentLoad(ctx, buckets[:count], failedBuckets)
|
||||
buckets = buckets[count:]
|
||||
}
|
||||
|
||||
if globalIsDistErasure {
|
||||
go sys.refreshBucketsMetadataLoop(ctx)
|
||||
go sys.refreshBucketsMetadataLoop(ctx, failedBuckets)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -575,7 +575,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
|
||||
if rule.Status == replication.Disabled {
|
||||
continue
|
||||
}
|
||||
clnt := globalBucketTargetSys.GetRemoteTargetClient(rule.Destination.Bucket)
|
||||
clnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, rule.Destination.Bucket)
|
||||
if clnt == nil {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, fmt.Errorf("replication config with rule ID %s has a stale target", rule.ID)), r.URL)
|
||||
return
|
||||
|
||||
+31
-25
@@ -113,7 +113,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
|
||||
if arn.Type != madmin.ReplicationService {
|
||||
return sameTarget, toAPIError(ctx, BucketRemoteArnTypeInvalid{Bucket: bucket})
|
||||
}
|
||||
clnt := globalBucketTargetSys.GetRemoteTargetClient(arnStr)
|
||||
clnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, arnStr)
|
||||
if clnt == nil {
|
||||
return sameTarget, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
|
||||
}
|
||||
@@ -138,7 +138,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
|
||||
}
|
||||
}
|
||||
// validate replication ARN against target endpoint
|
||||
c := globalBucketTargetSys.GetRemoteTargetClient(arnStr)
|
||||
c := globalBucketTargetSys.GetRemoteTargetClient(bucket, arnStr)
|
||||
if c != nil {
|
||||
if err := checkRemoteEndpoint(ctx, c.EndpointURL()); err != nil {
|
||||
switch err.(type) {
|
||||
@@ -280,7 +280,7 @@ func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplica
|
||||
}
|
||||
tgtArns := cfg.FilterTargetArns(opts)
|
||||
for _, tgtArn := range tgtArns {
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(tgtArn)
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn)
|
||||
// the target online status should not be used here while deciding
|
||||
// whether to replicate as the target could be temporarily down
|
||||
opts.TargetArn = tgtArn
|
||||
@@ -383,7 +383,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
|
||||
continue
|
||||
}
|
||||
}
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(tgtArn)
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn)
|
||||
// the target online status should not be used here while deciding
|
||||
// whether to replicate deletes as the target could be temporarily down
|
||||
tgtDsc := newReplicateTargetDecision(tgtArn, false, false)
|
||||
@@ -495,7 +495,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
|
||||
if dobj.TargetArn != "" && dobj.TargetArn != tgtEntry.Arn {
|
||||
continue
|
||||
}
|
||||
tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(tgtEntry.Arn)
|
||||
tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtEntry.Arn)
|
||||
if tgtClnt == nil {
|
||||
// Skip stale targets if any and log them to be missing atleast once.
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn)
|
||||
@@ -777,6 +777,7 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
|
||||
UserMetadata: meta,
|
||||
ContentType: objInfo.ContentType,
|
||||
ContentEncoding: objInfo.ContentEncoding,
|
||||
Expires: objInfo.Expires,
|
||||
StorageClass: sc,
|
||||
Internal: minio.AdvancedPutOptions{
|
||||
SourceVersionID: objInfo.VersionID,
|
||||
@@ -1023,7 +1024,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
|
||||
var wg sync.WaitGroup
|
||||
var mu sync.Mutex
|
||||
for _, tgtArn := range tgtArns {
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(tgtArn)
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn)
|
||||
if tgt == nil {
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn)
|
||||
sendEvent(eventArgs{
|
||||
@@ -2188,7 +2189,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
|
||||
return nil, oi, proxy
|
||||
}
|
||||
for _, t := range proxyTargets.Targets {
|
||||
tgt = globalBucketTargetSys.GetRemoteTargetClient(t.Arn)
|
||||
tgt = globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
|
||||
if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
|
||||
continue
|
||||
}
|
||||
@@ -2487,7 +2488,7 @@ func newresyncer() *replicationResyncer {
|
||||
}
|
||||
|
||||
// mark status of replication resync on remote target for the bucket
|
||||
func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts) {
|
||||
func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts, objAPI ObjectLayer) {
|
||||
s.Lock()
|
||||
defer s.Unlock()
|
||||
|
||||
@@ -2498,6 +2499,10 @@ func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpt
|
||||
m.TargetsMap[opts.arn] = st
|
||||
m.LastUpdate = UTCNow()
|
||||
s.statusMap[opts.bucket] = m
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
saveResyncStatus(ctx, opts.bucket, m, objAPI)
|
||||
}
|
||||
|
||||
// update replication resync stats for bucket's remote target
|
||||
@@ -2527,7 +2532,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
|
||||
resyncStatus := ResyncFailed
|
||||
defer func() {
|
||||
s.markStatus(resyncStatus, opts)
|
||||
s.markStatus(resyncStatus, opts, objectAPI)
|
||||
globalSiteResyncMetrics.incBucket(opts, resyncStatus)
|
||||
s.workerCh <- struct{}{}
|
||||
}()
|
||||
@@ -2556,14 +2561,14 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn))
|
||||
return
|
||||
}
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(opts.arn)
|
||||
tgt := globalBucketTargetSys.GetRemoteTargetClient(opts.bucket, opts.arn)
|
||||
if tgt == nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn))
|
||||
return
|
||||
}
|
||||
// mark resync status as resync started
|
||||
if !heal {
|
||||
s.markStatus(ResyncStarted, opts)
|
||||
s.markStatus(ResyncStarted, opts, objectAPI)
|
||||
}
|
||||
|
||||
// Walk through all object versions - Walk() is always in ascending order needed to ensure
|
||||
@@ -2634,16 +2639,18 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
|
||||
roi.EventType = ReplicateExisting
|
||||
replicateObject(ctx, roi, objectAPI)
|
||||
}
|
||||
_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
|
||||
|
||||
st := TargetReplicationResyncStatus{
|
||||
Object: roi.Name,
|
||||
Bucket: roi.Bucket,
|
||||
}
|
||||
|
||||
_, err := tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
|
||||
VersionID: roi.VersionID,
|
||||
Internal: minio.AdvancedGetOptions{
|
||||
ReplicationProxyRequest: "false",
|
||||
},
|
||||
})
|
||||
st := TargetReplicationResyncStatus{
|
||||
Object: roi.Name,
|
||||
Bucket: roi.Bucket,
|
||||
}
|
||||
if err != nil {
|
||||
if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
|
||||
st.ReplicatedCount++
|
||||
@@ -3181,12 +3188,16 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
|
||||
}
|
||||
}
|
||||
|
||||
func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntries, data []byte) {
|
||||
func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntries) {
|
||||
newReader := func() io.ReadCloser {
|
||||
r, w := io.Pipe()
|
||||
go func() {
|
||||
// Initialize MRF meta header.
|
||||
var data [4]byte
|
||||
binary.LittleEndian.PutUint16(data[0:2], mrfMetaFormat)
|
||||
binary.LittleEndian.PutUint16(data[2:4], mrfMetaVersion)
|
||||
mw := msgp.NewWriter(w)
|
||||
n, err := mw.Write(data)
|
||||
n, err := mw.Write(data[:])
|
||||
if err != nil {
|
||||
w.CloseWithError(err)
|
||||
return
|
||||
@@ -3224,15 +3235,10 @@ func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string
|
||||
|
||||
v := MRFReplicateEntries{
|
||||
Entries: entries,
|
||||
Version: mrfMetaVersionV1,
|
||||
Version: mrfMetaVersion,
|
||||
}
|
||||
data := make([]byte, 4, v.Msgsize()+4)
|
||||
|
||||
// Initialize the resync meta header.
|
||||
binary.LittleEndian.PutUint16(data[0:2], mrfMetaFormat)
|
||||
binary.LittleEndian.PutUint16(data[2:4], mrfMetaVersion)
|
||||
|
||||
p.persistToDrive(ctx, v, data)
|
||||
p.persistToDrive(ctx, v)
|
||||
}
|
||||
|
||||
// load mrf entries from disk
|
||||
|
||||
+87
-8
@@ -40,14 +40,29 @@ const (
|
||||
defaultHealthCheckReloadDuration = 30 * time.Minute
|
||||
)
|
||||
|
||||
type arnTarget struct {
|
||||
Client *TargetClient
|
||||
lastRefresh time.Time
|
||||
}
|
||||
|
||||
// arnErrs represents number of errors seen for a ARN and if update is in progress
|
||||
// to refresh remote targets from bucket metadata.
|
||||
type arnErrs struct {
|
||||
count int64
|
||||
updateInProgress bool
|
||||
bucket string
|
||||
}
|
||||
|
||||
// BucketTargetSys represents bucket targets subsystem
|
||||
type BucketTargetSys struct {
|
||||
sync.RWMutex
|
||||
arnRemotesMap map[string]*TargetClient
|
||||
arnRemotesMap map[string]arnTarget
|
||||
targetsMap map[string][]madmin.BucketTarget
|
||||
hMutex sync.RWMutex
|
||||
hc map[string]epHealth
|
||||
hcClient *madmin.AnonymousClient
|
||||
aMutex sync.RWMutex
|
||||
arnErrsMap map[string]arnErrs // map of ARN to error count of failures to get target
|
||||
}
|
||||
|
||||
type latencyStat struct {
|
||||
@@ -364,7 +379,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
|
||||
}
|
||||
|
||||
sys.targetsMap[bucket] = newtgts
|
||||
sys.arnRemotesMap[tgt.Arn] = clnt
|
||||
sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: clnt}
|
||||
sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
|
||||
return nil
|
||||
}
|
||||
@@ -432,11 +447,71 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
|
||||
return nil
|
||||
}
|
||||
|
||||
func (sys *BucketTargetSys) markRefreshInProgress(bucket, arn string) {
|
||||
sys.aMutex.Lock()
|
||||
defer sys.aMutex.Unlock()
|
||||
if v, ok := sys.arnErrsMap[arn]; !ok {
|
||||
sys.arnErrsMap[arn] = arnErrs{
|
||||
updateInProgress: true,
|
||||
count: v.count + 1,
|
||||
bucket: bucket,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketTargetSys) markRefreshDone(bucket, arn string) {
|
||||
sys.aMutex.Lock()
|
||||
defer sys.aMutex.Unlock()
|
||||
if v, ok := sys.arnErrsMap[arn]; ok {
|
||||
sys.arnErrsMap[arn] = arnErrs{
|
||||
updateInProgress: false,
|
||||
count: v.count,
|
||||
bucket: bucket,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (sys *BucketTargetSys) isReloadingTarget(bucket, arn string) bool {
|
||||
sys.aMutex.RLock()
|
||||
defer sys.aMutex.RUnlock()
|
||||
if v, ok := sys.arnErrsMap[arn]; ok {
|
||||
return v.updateInProgress
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (sys *BucketTargetSys) incTargetErr(bucket, arn string) {
|
||||
sys.aMutex.Lock()
|
||||
defer sys.aMutex.Unlock()
|
||||
if v, ok := sys.arnErrsMap[arn]; ok {
|
||||
sys.arnErrsMap[arn] = arnErrs{
|
||||
updateInProgress: v.updateInProgress,
|
||||
count: v.count + 1,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// GetRemoteTargetClient returns minio-go client for replication target instance
|
||||
func (sys *BucketTargetSys) GetRemoteTargetClient(arn string) *TargetClient {
|
||||
func (sys *BucketTargetSys) GetRemoteTargetClient(bucket, arn string) *TargetClient {
|
||||
sys.RLock()
|
||||
defer sys.RUnlock()
|
||||
return sys.arnRemotesMap[arn]
|
||||
tgt := sys.arnRemotesMap[arn]
|
||||
sys.RUnlock()
|
||||
|
||||
if tgt.Client != nil {
|
||||
return tgt.Client
|
||||
}
|
||||
defer func() { // lazy refresh remote targets
|
||||
if tgt.Client == nil && !sys.isReloadingTarget(bucket, arn) && (tgt.lastRefresh.Equal(timeSentinel) || tgt.lastRefresh.Before(UTCNow().Add(-5*time.Minute))) {
|
||||
tgts, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
|
||||
if err == nil {
|
||||
sys.markRefreshInProgress(bucket, arn)
|
||||
sys.UpdateAllTargets(bucket, tgts)
|
||||
sys.markRefreshDone(bucket, arn)
|
||||
}
|
||||
}
|
||||
sys.incTargetErr(bucket, arn)
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetRemoteBucketTargetByArn returns BucketTarget for a ARN
|
||||
@@ -457,8 +532,9 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
|
||||
// NewBucketTargetSys - creates new replication system.
|
||||
func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
|
||||
sys := &BucketTargetSys{
|
||||
arnRemotesMap: make(map[string]*TargetClient),
|
||||
arnRemotesMap: make(map[string]arnTarget),
|
||||
targetsMap: make(map[string][]madmin.BucketTarget),
|
||||
arnErrsMap: make(map[string]arnErrs),
|
||||
hc: make(map[string]epHealth),
|
||||
hcClient: newHCClient(),
|
||||
}
|
||||
@@ -502,7 +578,10 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sys.arnRemotesMap[tgt.Arn] = tgtClient
|
||||
sys.arnRemotesMap[tgt.Arn] = arnTarget{
|
||||
Client: tgtClient,
|
||||
lastRefresh: UTCNow(),
|
||||
}
|
||||
sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
|
||||
}
|
||||
|
||||
@@ -526,7 +605,7 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
|
||||
logger.LogIf(GlobalContext, err)
|
||||
continue
|
||||
}
|
||||
sys.arnRemotesMap[tgt.Arn] = tgtClient
|
||||
sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: tgtClient}
|
||||
sys.updateBandwidthLimit(bucket.Name, tgt.Arn, tgt.BandwidthLimit)
|
||||
}
|
||||
sys.targetsMap[bucket.Name] = cfg.Targets
|
||||
|
||||
+1
-1
@@ -131,7 +131,7 @@ func performCallhome(ctx context.Context) {
|
||||
Version: madmin.HealthInfoVersion,
|
||||
Minio: madmin.MinioHealthInfo{
|
||||
Info: madmin.MinioInfo{
|
||||
DeploymentID: globalDeploymentID,
|
||||
DeploymentID: globalDeploymentID(),
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
+8
-7
@@ -122,8 +122,8 @@ func init() {
|
||||
const consolePrefix = "CONSOLE_"
|
||||
|
||||
func minioConfigToConsoleFeatures() {
|
||||
os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID)
|
||||
os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID)
|
||||
os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID())
|
||||
os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID())
|
||||
if globalMinioEndpoint != "" {
|
||||
os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
|
||||
} else {
|
||||
@@ -167,9 +167,12 @@ func minioConfigToConsoleFeatures() {
|
||||
if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" {
|
||||
os.Setenv("CONSOLE_ANIMATED_LOGIN", value)
|
||||
}
|
||||
|
||||
// Pass on the session duration environment variable, else we will default to 12 hours
|
||||
if value := env.Get(config.EnvBrowserSessionDuration, ""); value != "" {
|
||||
os.Setenv("CONSOLE_STS_DURATION", value)
|
||||
if valueSts := env.Get(config.EnvMinioStsDuration, ""); valueSts != "" {
|
||||
os.Setenv("CONSOLE_STS_DURATION", valueSts)
|
||||
} else if valueSession := env.Get(config.EnvBrowserSessionDuration, ""); valueSession != "" {
|
||||
os.Setenv("CONSOLE_STS_DURATION", valueSession)
|
||||
}
|
||||
|
||||
os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
|
||||
@@ -191,7 +194,7 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
|
||||
DisplayName: cfg.DisplayName,
|
||||
ClientID: cfg.ClientID,
|
||||
ClientSecret: cfg.ClientSecret,
|
||||
HMACSalt: globalDeploymentID,
|
||||
HMACSalt: globalDeploymentID(),
|
||||
HMACPassphrase: cfg.ClientID,
|
||||
Scopes: strings.Join(cfg.DiscoveryDoc.ScopesSupported, ","),
|
||||
Userinfo: cfg.ClaimUserinfo,
|
||||
@@ -619,8 +622,6 @@ func loadEnvVarsFromFiles() {
|
||||
}
|
||||
|
||||
func handleCommonEnvVars() {
|
||||
loadEnvVarsFromFiles()
|
||||
|
||||
var err error
|
||||
globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
|
||||
if err != nil {
|
||||
|
||||
@@ -27,7 +27,6 @@ import (
|
||||
"github.com/minio/madmin-go/v3"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/api"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
"github.com/minio/minio/internal/config/callhome"
|
||||
"github.com/minio/minio/internal/config/compress"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
@@ -46,7 +45,6 @@ import (
|
||||
"github.com/minio/minio/internal/config/subnet"
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v2/env"
|
||||
)
|
||||
@@ -54,7 +52,6 @@ import (
|
||||
func initHelp() {
|
||||
kvs := map[string]config.KVS{
|
||||
config.EtcdSubSys: etcd.DefaultKVS,
|
||||
config.CacheSubSys: cache.DefaultKVS,
|
||||
config.CompressionSubSys: compress.DefaultKVS,
|
||||
config.IdentityLDAPSubSys: xldap.DefaultKVS,
|
||||
config.IdentityOpenIDSubSys: openid.DefaultKVS,
|
||||
@@ -209,10 +206,6 @@ func initHelp() {
|
||||
Key: config.EtcdSubSys,
|
||||
Description: "persist IAM assets externally to etcd",
|
||||
},
|
||||
config.HelpKV{
|
||||
Key: config.CacheSubSys,
|
||||
Description: "[DEPRECATED] add caching storage tier",
|
||||
},
|
||||
}
|
||||
|
||||
if globalIsErasure {
|
||||
@@ -232,7 +225,6 @@ func initHelp() {
|
||||
config.APISubSys: api.Help,
|
||||
config.StorageClassSubSys: storageclass.Help,
|
||||
config.EtcdSubSys: etcd.Help,
|
||||
config.CacheSubSys: cache.Help,
|
||||
config.CompressionSubSys: compress.Help,
|
||||
config.HealSubSys: heal.Help,
|
||||
config.ScannerSubSys: scanner.Help,
|
||||
@@ -302,10 +294,6 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
|
||||
return err
|
||||
}
|
||||
}
|
||||
case config.CacheSubSys:
|
||||
if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
|
||||
return err
|
||||
}
|
||||
case config.CompressionSubSys:
|
||||
if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
|
||||
return err
|
||||
@@ -493,20 +481,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
|
||||
logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
|
||||
}
|
||||
|
||||
globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
|
||||
}
|
||||
|
||||
if globalCacheConfig.Enabled {
|
||||
if cacheEncKey := env.Get(cache.EnvCacheEncryptionKey, ""); cacheEncKey != "" {
|
||||
globalCacheKMS, err = kms.Parse(cacheEncKey)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to setup encryption cache: %w", err))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
globalAutoEncryption = crypto.LookupAutoEncryption() // Enable auto-encryption if enabled
|
||||
if globalAutoEncryption && GlobalKMS == nil {
|
||||
logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,336 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Test if config v1 is purged
|
||||
func TestServerConfigMigrateV1(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(fsDir)
|
||||
err = newTestConfig(globalMinioDefaultRegion, objLayer)
|
||||
if err != nil {
|
||||
t.Fatalf("Init Test config failed")
|
||||
}
|
||||
rootPath := t.TempDir()
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
globalObjLayerMutex.Lock()
|
||||
globalObjectAPI = objLayer
|
||||
globalObjLayerMutex.Unlock()
|
||||
|
||||
// Create a V1 config json file and store it
|
||||
configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
|
||||
configPath := rootPath + "/fsUsers.json"
|
||||
if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Fire a migrateConfig()
|
||||
if err := migrateConfig(); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Check if config v1 is removed from filesystem
|
||||
if _, err := os.Stat(configPath); err == nil || !osIsNotExist(err) {
|
||||
t.Fatal("Config V1 file is not purged")
|
||||
}
|
||||
|
||||
// Initialize server config and check again if everything is fine
|
||||
if err := loadConfig(objLayer, nil); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test if all migrate code returns nil when config file does not
|
||||
// exist
|
||||
func TestServerConfigMigrateInexistentConfig(t *testing.T) {
|
||||
globalConfigDir = &ConfigDir{path: t.TempDir()}
|
||||
|
||||
if err := migrateV2ToV3(); err != nil {
|
||||
t.Fatal("migrate v2 to v3 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV3ToV4(); err != nil {
|
||||
t.Fatal("migrate v3 to v4 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV4ToV5(); err != nil {
|
||||
t.Fatal("migrate v4 to v5 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV5ToV6(); err != nil {
|
||||
t.Fatal("migrate v5 to v6 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV6ToV7(); err != nil {
|
||||
t.Fatal("migrate v6 to v7 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV7ToV8(); err != nil {
|
||||
t.Fatal("migrate v7 to v8 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV8ToV9(); err != nil {
|
||||
t.Fatal("migrate v8 to v9 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV9ToV10(); err != nil {
|
||||
t.Fatal("migrate v9 to v10 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV10ToV11(); err != nil {
|
||||
t.Fatal("migrate v10 to v11 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV11ToV12(); err != nil {
|
||||
t.Fatal("migrate v11 to v12 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV12ToV13(); err != nil {
|
||||
t.Fatal("migrate v12 to v13 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV13ToV14(); err != nil {
|
||||
t.Fatal("migrate v13 to v14 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV14ToV15(); err != nil {
|
||||
t.Fatal("migrate v14 to v15 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV15ToV16(); err != nil {
|
||||
t.Fatal("migrate v15 to v16 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV16ToV17(); err != nil {
|
||||
t.Fatal("migrate v16 to v17 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV17ToV18(); err != nil {
|
||||
t.Fatal("migrate v17 to v18 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV18ToV19(); err != nil {
|
||||
t.Fatal("migrate v18 to v19 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV19ToV20(); err != nil {
|
||||
t.Fatal("migrate v19 to v20 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV20ToV21(); err != nil {
|
||||
t.Fatal("migrate v20 to v21 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV21ToV22(); err != nil {
|
||||
t.Fatal("migrate v21 to v22 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV22ToV23(); err != nil {
|
||||
t.Fatal("migrate v22 to v23 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV23ToV24(); err != nil {
|
||||
t.Fatal("migrate v23 to v24 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV24ToV25(); err != nil {
|
||||
t.Fatal("migrate v24 to v25 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV25ToV26(); err != nil {
|
||||
t.Fatal("migrate v25 to v26 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV26ToV27(); err != nil {
|
||||
t.Fatal("migrate v26 to v27 should succeed when no config file is found")
|
||||
}
|
||||
if err := migrateV27ToV28(); err != nil {
|
||||
t.Fatal("migrate v27 to v28 should succeed when no config file is found")
|
||||
}
|
||||
}
|
||||
|
||||
// Test if a config migration from v2 to v33 is successfully done
|
||||
func TestServerConfigMigrateV2toV33(t *testing.T) {
|
||||
rootPath := t.TempDir()
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
|
||||
objLayer, fsDir, err := prepareFS(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer os.RemoveAll(fsDir)
|
||||
|
||||
configPath := rootPath + SlashSeparator + minioConfigFile
|
||||
|
||||
// Create a corrupted config file
|
||||
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
// Fire a migrateConfig()
|
||||
if err := migrateConfig(); err == nil {
|
||||
t.Fatal("migration should fail with corrupted config file")
|
||||
}
|
||||
|
||||
accessKey := "accessfoo"
|
||||
secretKey := "secretfoo"
|
||||
|
||||
// Create a V2 config json file and store it
|
||||
configJSON := "{ \"version\":\"2\", \"credentials\": {\"accessKeyId\":\"" + accessKey + "\", \"secretAccessKey\":\"" + secretKey + "\", \"region\":\"us-east-1\"}, \"mongoLogger\":{\"addr\":\"127.0.0.1:3543\", \"db\":\"foodb\", \"collection\":\"foo\"}, \"syslogLogger\":{\"network\":\"127.0.0.1:543\", \"addr\":\"addr\"}, \"fileLogger\":{\"filename\":\"log.out\"}}"
|
||||
if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Fire a migrateConfig()
|
||||
if err := migrateConfig(); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
if err := migrateConfigToMinioSys(objLayer); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
srvCfg, err := readConfigWithoutMigrate(context.Background(), objLayer)
|
||||
if err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
if err = saveServerConfig(GlobalContext, objLayer, srvCfg); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Initialize server config and check again if everything is fine
|
||||
if err := loadConfig(objLayer, nil); err != nil {
|
||||
t.Fatalf("Unable to initialize from updated config file %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
func TestServerConfigMigrateFaultyConfig(t *testing.T) {
|
||||
rootPath := t.TempDir()
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
configPath := rootPath + SlashSeparator + minioConfigFile
|
||||
|
||||
// Create a corrupted config file
|
||||
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Test different migrate versions and be sure they are returning an error
|
||||
if err := migrateV2ToV3(); err == nil {
|
||||
t.Fatal("migrateConfigV2ToV3() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV3ToV4(); err == nil {
|
||||
t.Fatal("migrateConfigV3ToV4() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV4ToV5(); err == nil {
|
||||
t.Fatal("migrateConfigV4ToV5() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV5ToV6(); err == nil {
|
||||
t.Fatal("migrateConfigV5ToV6() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV6ToV7(); err == nil {
|
||||
t.Fatal("migrateConfigV6ToV7() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV7ToV8(); err == nil {
|
||||
t.Fatal("migrateConfigV7ToV8() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV8ToV9(); err == nil {
|
||||
t.Fatal("migrateConfigV8ToV9() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV9ToV10(); err == nil {
|
||||
t.Fatal("migrateConfigV9ToV10() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV10ToV11(); err == nil {
|
||||
t.Fatal("migrateConfigV10ToV11() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV11ToV12(); err == nil {
|
||||
t.Fatal("migrateConfigV11ToV12() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV12ToV13(); err == nil {
|
||||
t.Fatal("migrateConfigV12ToV13() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV13ToV14(); err == nil {
|
||||
t.Fatal("migrateConfigV13ToV14() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV14ToV15(); err == nil {
|
||||
t.Fatal("migrateConfigV14ToV15() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV15ToV16(); err == nil {
|
||||
t.Fatal("migrateConfigV15ToV16() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV16ToV17(); err == nil {
|
||||
t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV17ToV18(); err == nil {
|
||||
t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV18ToV19(); err == nil {
|
||||
t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV19ToV20(); err == nil {
|
||||
t.Fatal("migrateConfigV19ToV20() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV20ToV21(); err == nil {
|
||||
t.Fatal("migrateConfigV20ToV21() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV21ToV22(); err == nil {
|
||||
t.Fatal("migrateConfigV21ToV22() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV22ToV23(); err == nil {
|
||||
t.Fatal("migrateConfigV22ToV23() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV23ToV24(); err == nil {
|
||||
t.Fatal("migrateConfigV23ToV24() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV24ToV25(); err == nil {
|
||||
t.Fatal("migrateConfigV24ToV25() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV25ToV26(); err == nil {
|
||||
t.Fatal("migrateConfigV25ToV26() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV26ToV27(); err == nil {
|
||||
t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
|
||||
}
|
||||
if err := migrateV27ToV28(); err == nil {
|
||||
t.Fatal("migrateConfigV27ToV28() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
// Test if all migrate code returns error with corrupted config files
|
||||
func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
|
||||
rootPath := t.TempDir()
|
||||
|
||||
globalConfigDir = &ConfigDir{path: rootPath}
|
||||
configPath := rootPath + SlashSeparator + minioConfigFile
|
||||
|
||||
for i := 3; i <= 17; i++ {
|
||||
// Create a corrupted config file
|
||||
if err := os.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
|
||||
0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Test different migrate versions and be sure they are returning an error
|
||||
if err := migrateConfig(); err == nil {
|
||||
t.Fatal("migrateConfig() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
|
||||
// Create a corrupted config file for version '2'.
|
||||
if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
|
||||
t.Fatal("Unexpected error: ", err)
|
||||
}
|
||||
|
||||
// Test different migrate versions and be sure they are returning an error
|
||||
if err := migrateConfig(); err == nil {
|
||||
t.Fatal("migrateConfig() should fail with a corrupted json")
|
||||
}
|
||||
}
|
||||
@@ -18,418 +18,18 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
"github.com/minio/minio/internal/config/compress"
|
||||
xldap "github.com/minio/minio/internal/config/identity/ldap"
|
||||
"github.com/minio/minio/internal/config/identity/openid"
|
||||
"github.com/minio/minio/internal/config/notify"
|
||||
"github.com/minio/minio/internal/config/policy/opa"
|
||||
"github.com/minio/minio/internal/config/storageclass"
|
||||
"github.com/minio/minio/internal/event/target"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v2/quick"
|
||||
)
|
||||
|
||||
type configV1 struct {
|
||||
Version string `json:"version"`
|
||||
AccessKey string `json:"accessKeyId"`
|
||||
SecretKey string `json:"secretAccessKey"`
|
||||
}
|
||||
|
||||
type configV2 struct {
|
||||
Version string `json:"version"`
|
||||
Credentials struct {
|
||||
AccessKey string `json:"accessKeyId"`
|
||||
SecretKey string `json:"secretAccessKey"`
|
||||
Region string `json:"region"`
|
||||
} `json:"credentials"`
|
||||
MongoLogger struct {
|
||||
Addr string `json:"addr"`
|
||||
DB string `json:"db"`
|
||||
Collection string `json:"collection"`
|
||||
} `json:"mongoLogger"`
|
||||
SyslogLogger struct {
|
||||
Network string `json:"network"`
|
||||
Addr string `json:"addr"`
|
||||
} `json:"syslogLogger"`
|
||||
FileLogger struct {
|
||||
Filename string `json:"filename"`
|
||||
} `json:"fileLogger"`
|
||||
}
|
||||
|
||||
// backendV3 type.
|
||||
type backendV3 struct {
|
||||
Type string `json:"type"`
|
||||
Disk string `json:"disk,omitempty"`
|
||||
Disks []string `json:"disks,omitempty"`
|
||||
}
|
||||
|
||||
// syslogLogger v3
|
||||
type syslogLoggerV3 struct {
|
||||
Enable bool `json:"enable"`
|
||||
Addr string `json:"address"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
// loggerV3 type.
|
||||
type loggerV3 struct {
|
||||
Console struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
File struct {
|
||||
Enable bool `json:"enable"`
|
||||
Filename string `json:"fileName"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
Syslog struct {
|
||||
Enable bool `json:"enable"`
|
||||
Addr string `json:"address"`
|
||||
Level string `json:"level"`
|
||||
} `json:"syslog"`
|
||||
// Add new loggers here.
|
||||
}
|
||||
|
||||
// configV3 server configuration version '3'.
|
||||
type configV3 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// Backend configuration.
|
||||
Backend backendV3 `json:"backend"`
|
||||
|
||||
// http Server configuration.
|
||||
Addr string `json:"address"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV3 `json:"logger"`
|
||||
}
|
||||
|
||||
// logger type representing version '4' logger config.
|
||||
type loggerV4 struct {
|
||||
Console struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
} `json:"console"`
|
||||
File struct {
|
||||
Enable bool `json:"enable"`
|
||||
Filename string `json:"fileName"`
|
||||
Level string `json:"level"`
|
||||
} `json:"file"`
|
||||
Syslog struct {
|
||||
Enable bool `json:"enable"`
|
||||
Addr string `json:"address"`
|
||||
Level string `json:"level"`
|
||||
} `json:"syslog"`
|
||||
}
|
||||
|
||||
// configV4 server configuration version '4'.
|
||||
type configV4 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV4 `json:"logger"`
|
||||
}
|
||||
|
||||
// logger type representing version '5' logger config.
|
||||
type loggerV5 struct {
|
||||
Console struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
} `json:"console"`
|
||||
File struct {
|
||||
Enable bool `json:"enable"`
|
||||
Filename string `json:"fileName"`
|
||||
Level string `json:"level"`
|
||||
} `json:"file"`
|
||||
Syslog struct {
|
||||
Enable bool `json:"enable"`
|
||||
Addr string `json:"address"`
|
||||
Level string `json:"level"`
|
||||
} `json:"syslog"`
|
||||
AMQP struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
URL string `json:"url"`
|
||||
Exchange string `json:"exchange"`
|
||||
RoutingKey string `json:"routingKey"`
|
||||
ExchangeType string `json:"exchangeType"`
|
||||
Mandatory bool `json:"mandatory"`
|
||||
Immediate bool `json:"immediate"`
|
||||
Durable bool `json:"durable"`
|
||||
Internal bool `json:"internal"`
|
||||
NoWait bool `json:"noWait"`
|
||||
AutoDeleted bool `json:"autoDeleted"`
|
||||
} `json:"amqp"`
|
||||
ElasticSearch struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
URL string `json:"url"`
|
||||
Index string `json:"index"`
|
||||
} `json:"elasticsearch"`
|
||||
Redis struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
Addr string `json:"address"`
|
||||
Password string `json:"password"`
|
||||
Key string `json:"key"`
|
||||
} `json:"redis"`
|
||||
}
|
||||
|
||||
// configV5 server configuration version '5'.
|
||||
type configV5 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV5 `json:"logger"`
|
||||
}
|
||||
|
||||
// consoleLogger - default logger if not other logging is enabled.
|
||||
type consoleLoggerV1 struct {
|
||||
Enable bool `json:"enable"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
type fileLoggerV1 struct {
|
||||
Enable bool `json:"enable"`
|
||||
Filename string `json:"fileName"`
|
||||
Level string `json:"level"`
|
||||
}
|
||||
|
||||
type loggerV6 struct {
|
||||
Console consoleLoggerV1 `json:"console"`
|
||||
File fileLoggerV1 `json:"file"`
|
||||
Syslog syslogLoggerV3 `json:"syslog"`
|
||||
}
|
||||
|
||||
// configV6 server configuration version '6'.
|
||||
type configV6 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
// Notifier represents collection of supported notification queues in version
|
||||
// 1 without NATS streaming.
|
||||
type notifierV1 struct {
|
||||
AMQP map[string]target.AMQPArgs `json:"amqp"`
|
||||
NATS map[string]natsNotifyV1 `json:"nats"`
|
||||
ElasticSearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
|
||||
Redis map[string]target.RedisArgs `json:"redis"`
|
||||
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
|
||||
Kafka map[string]target.KafkaArgs `json:"kafka"`
|
||||
}
|
||||
|
||||
// Notifier represents collection of supported notification queues in version 2
|
||||
// with NATS streaming but without webhook.
|
||||
type notifierV2 struct {
|
||||
AMQP map[string]target.AMQPArgs `json:"amqp"`
|
||||
NATS map[string]target.NATSArgs `json:"nats"`
|
||||
ElasticSearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
|
||||
Redis map[string]target.RedisArgs `json:"redis"`
|
||||
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
|
||||
Kafka map[string]target.KafkaArgs `json:"kafka"`
|
||||
}
|
||||
|
||||
// configV7 server configuration version '7'.
|
||||
type serverConfigV7 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV8 server configuration version '8'. Adds NATS notify.Config
|
||||
// configuration.
|
||||
type serverConfigV8 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV9 server configuration version '9'. Adds PostgreSQL
|
||||
// notify.Config configuration.
|
||||
type serverConfigV9 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV6 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
type loggerV7 struct {
|
||||
sync.RWMutex
|
||||
Console consoleLoggerV1 `json:"console"`
|
||||
File fileLoggerV1 `json:"file"`
|
||||
}
|
||||
|
||||
// serverConfigV10 server configuration version '10' which is like
|
||||
// version '9' except it drops support of syslog config, and makes the
|
||||
// RWMutex global (so it does not exist in this struct).
|
||||
type serverConfigV10 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
// natsNotifyV1 - structure was valid until config V 11
|
||||
type natsNotifyV1 struct {
|
||||
Enable bool `json:"enable"`
|
||||
Address string `json:"address"`
|
||||
Subject string `json:"subject"`
|
||||
Username string `json:"username"`
|
||||
Password string `json:"password"`
|
||||
Token string `json:"token"`
|
||||
Secure bool `json:"secure"`
|
||||
PingInterval int64 `json:"pingInterval"`
|
||||
}
|
||||
|
||||
// serverConfigV11 server configuration version '11' which is like
|
||||
// version '10' except it adds support for Kafka notifications.
|
||||
type serverConfigV11 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV1 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV12 server configuration version '12' which is like
|
||||
// version '11' except it adds support for NATS streaming notifications.
|
||||
type serverConfigV12 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV2 `json:"notify"`
|
||||
}
|
||||
|
||||
type notifierV3 struct {
|
||||
AMQP map[string]target.AMQPArgs `json:"amqp"`
|
||||
Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
|
||||
Kafka map[string]target.KafkaArgs `json:"kafka"`
|
||||
MQTT map[string]target.MQTTArgs `json:"mqtt"`
|
||||
MySQL map[string]target.MySQLArgs `json:"mysql"`
|
||||
NATS map[string]target.NATSArgs `json:"nats"`
|
||||
PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"`
|
||||
Redis map[string]target.RedisArgs `json:"redis"`
|
||||
Webhook map[string]target.WebhookArgs `json:"webhook"`
|
||||
}
|
||||
|
||||
// serverConfigV13 server configuration version '13' which is like
|
||||
// version '12' except it adds support for webhook notification.
|
||||
type serverConfigV13 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV14 server configuration version '14' which is like
|
||||
// version '13' except it adds support of browser param.
|
||||
type serverConfigV14 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV15 server configuration version '15' which is like
|
||||
// version '14' except it adds mysql support
|
||||
type serverConfigV15 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggerV7 `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// FileLogger is introduced to workaround the dependency about logrus
|
||||
type FileLogger struct {
|
||||
Enable bool `json:"enable"`
|
||||
@@ -441,282 +41,6 @@ type ConsoleLogger struct {
|
||||
Enable bool `json:"enable"`
|
||||
}
|
||||
|
||||
// Loggers struct is defined with FileLogger and ConsoleLogger
|
||||
// although they are removed from logging logic. They are
|
||||
// kept here just to workaround the dependency migration
|
||||
// code/logic has on them.
|
||||
type loggers struct {
|
||||
sync.RWMutex
|
||||
Console ConsoleLogger `json:"console"`
|
||||
File FileLogger `json:"file"`
|
||||
}
|
||||
|
||||
// serverConfigV16 server configuration version '16' which is like
|
||||
// version '15' except it makes a change to logging configuration.
|
||||
type serverConfigV16 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV17 server configuration version '17' which is like
|
||||
// version '16' except it adds support for "format" parameter in
|
||||
// database event notification targets: PostgreSQL, MySQL, Redis and
|
||||
// Elasticsearch.
|
||||
type serverConfigV17 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV18 server configuration version '18' which is like
|
||||
// version '17' except it adds support for "deliveryMode" parameter in
|
||||
// the AMQP notification target.
|
||||
type serverConfigV18 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV19 server configuration version '19' which is like
|
||||
// version '18' except it adds support for MQTT notifications.
|
||||
type serverConfigV19 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV20 server configuration version '20' which is like
|
||||
// version '19' except it adds support for VirtualHostDomain
|
||||
type serverConfigV20 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Additional error logging configuration.
|
||||
Logger *loggers `json:"logger"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV21 is just like version '20' without logger field
|
||||
type serverConfigV21 struct {
|
||||
sync.RWMutex
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify *notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV22 is just like version '21' with added support
|
||||
// for StorageClass.
|
||||
type serverConfigV22 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV23 is just like version '22' with addition of cache field.
|
||||
type serverConfigV23 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache cache.Config `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV24 is just like version '23', we had to revert
|
||||
// the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
|
||||
type serverConfigV24 struct {
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache cache.Config `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV25 is just like version '24', stores additionally
|
||||
// worm variable.
|
||||
type serverConfigV25 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Worm config.BoolFlag `json:"worm"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache cache.Config `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV26 is just like version '25', stores additionally
|
||||
// cache max use value in 'cache.Config'.
|
||||
type serverConfigV26 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Worm config.BoolFlag `json:"worm"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache cache.Config `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
}
|
||||
|
||||
// serverConfigV27 is just like version '26', stores additionally
|
||||
// the logger field
|
||||
type serverConfigV27 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Browser config.BoolFlag `json:"browser"`
|
||||
Worm config.BoolFlag `json:"worm"`
|
||||
Domain string `json:"domain"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache cache.Config `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger logger.Config `json:"logger"`
|
||||
}
|
||||
|
||||
// serverConfigV28 is just like version '27', additionally
|
||||
// storing KMS config
|
||||
type serverConfigV28 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
|
||||
Version string `json:"version"`
|
||||
|
||||
// S3 API configuration.
|
||||
Credential auth.Credentials `json:"credential"`
|
||||
Region string `json:"region"`
|
||||
Worm config.BoolFlag `json:"worm"`
|
||||
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache cache.Config `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notifierV3 `json:"notify"`
|
||||
|
||||
// Logger configuration
|
||||
Logger logger.Config `json:"logger"`
|
||||
}
|
||||
|
||||
// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets.
|
||||
type serverConfigV33 struct {
|
||||
quick.Config `json:"-"` // ignore interfaces
|
||||
@@ -731,9 +55,6 @@ type serverConfigV33 struct {
|
||||
// Storage class configuration
|
||||
StorageClass storageclass.Config `json:"storageclass"`
|
||||
|
||||
// Cache configuration
|
||||
Cache cache.Config `json:"cache"`
|
||||
|
||||
// Notification queue configuration.
|
||||
Notify notify.Config `json:"notify"`
|
||||
|
||||
|
||||
+1
-2
@@ -343,7 +343,6 @@ func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, ca
|
||||
// No useful information...
|
||||
return cache, err
|
||||
}
|
||||
|
||||
s.newCache.Info.LastUpdate = UTCNow()
|
||||
s.newCache.Info.NextCycle = cache.Info.NextCycle
|
||||
return s.newCache, nil
|
||||
@@ -1174,7 +1173,7 @@ func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
|
||||
if obj.DeleteMarker {
|
||||
return false
|
||||
}
|
||||
globalTransitionState.queueTransitionTask(obj, event, src)
|
||||
globalTransitionState.queueTransitionTask(obj, event, src, false)
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
+27
-34
@@ -78,8 +78,8 @@ func newAllTierStats() *allTierStats {
|
||||
}
|
||||
}
|
||||
|
||||
func (ats *allTierStats) addSizes(sz sizeSummary) {
|
||||
for tier, st := range sz.tiers {
|
||||
func (ats *allTierStats) addSizes(tiers map[string]tierStats) {
|
||||
for tier, st := range tiers {
|
||||
ats.Tiers[tier] = ats.Tiers[tier].add(st)
|
||||
}
|
||||
}
|
||||
@@ -95,18 +95,16 @@ func (ats *allTierStats) clone() *allTierStats {
|
||||
return nil
|
||||
}
|
||||
dst := *ats
|
||||
if dst.Tiers != nil {
|
||||
dst.Tiers = make(map[string]tierStats, len(dst.Tiers))
|
||||
for tier, st := range dst.Tiers {
|
||||
dst.Tiers[tier] = st
|
||||
}
|
||||
dst.Tiers = make(map[string]tierStats, len(ats.Tiers))
|
||||
for tier, st := range ats.Tiers {
|
||||
dst.Tiers[tier] = st
|
||||
}
|
||||
return &dst
|
||||
}
|
||||
|
||||
func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats {
|
||||
func (ats *allTierStats) populateStats(stats map[string]madmin.TierStats) {
|
||||
if ats == nil {
|
||||
return stats
|
||||
return
|
||||
}
|
||||
|
||||
// Update stats for tiers as they become available.
|
||||
@@ -117,7 +115,7 @@ func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[strin
|
||||
NumObjects: st.NumObjects,
|
||||
}
|
||||
}
|
||||
return stats
|
||||
return
|
||||
}
|
||||
|
||||
// tierStats holds per-tier stats of a remote tier.
|
||||
@@ -128,10 +126,11 @@ type tierStats struct {
|
||||
}
|
||||
|
||||
func (ts tierStats) add(u tierStats) tierStats {
|
||||
ts.TotalSize += u.TotalSize
|
||||
ts.NumVersions += u.NumVersions
|
||||
ts.NumObjects += u.NumObjects
|
||||
return ts
|
||||
return tierStats{
|
||||
TotalSize: ts.TotalSize + u.TotalSize,
|
||||
NumVersions: ts.NumVersions + u.NumVersions,
|
||||
NumObjects: ts.NumObjects + u.NumObjects,
|
||||
}
|
||||
}
|
||||
|
||||
//msgp:tuple replicationStatsV1
|
||||
@@ -197,12 +196,11 @@ func (r *replicationAllStats) clone() *replicationAllStats {
|
||||
dst := *r
|
||||
|
||||
// Copy individual targets.
|
||||
if dst.Targets != nil {
|
||||
dst.Targets = make(map[string]replicationStats, len(dst.Targets))
|
||||
for k, v := range r.Targets {
|
||||
dst.Targets[k] = v
|
||||
}
|
||||
dst.Targets = make(map[string]replicationStats, len(r.Targets))
|
||||
for k, v := range r.Targets {
|
||||
dst.Targets[k] = v
|
||||
}
|
||||
|
||||
return &dst
|
||||
}
|
||||
|
||||
@@ -360,11 +358,11 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
|
||||
tgtStat.PendingCount += st.pendingCount
|
||||
e.ReplicationStats.Targets[arn] = tgtStat
|
||||
}
|
||||
if summary.tiers != nil {
|
||||
if len(summary.tiers) != 0 {
|
||||
if e.AllTierStats == nil {
|
||||
e.AllTierStats = newAllTierStats()
|
||||
}
|
||||
e.AllTierStats.addSizes(summary)
|
||||
e.AllTierStats.addSizes(summary.tiers)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -403,7 +401,7 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
|
||||
e.ObjVersions[i] += v
|
||||
}
|
||||
|
||||
if other.AllTierStats != nil {
|
||||
if other.AllTierStats != nil && len(other.AllTierStats.Tiers) != 0 {
|
||||
if e.AllTierStats == nil {
|
||||
e.AllTierStats = newAllTierStats()
|
||||
}
|
||||
@@ -987,15 +985,10 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
return ctx.Err()
|
||||
case maxConcurrentScannerSaves <- struct{}{}:
|
||||
}
|
||||
defer func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case <-maxConcurrentScannerSaves:
|
||||
}
|
||||
}()
|
||||
|
||||
buf := bytebufferpool.Get()
|
||||
defer func() {
|
||||
<-maxConcurrentScannerSaves
|
||||
buf.Reset()
|
||||
bytebufferpool.Put(buf)
|
||||
}()
|
||||
@@ -1004,12 +997,12 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
return err
|
||||
}
|
||||
|
||||
hr, err := hash.NewReader(ctx, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
save := func(name string, timeout time.Duration) error {
|
||||
hr, err := hash.NewReader(ctx, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len()))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Abandon if more than a minute, so we don't hold up scanner.
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
@@ -1024,7 +1017,7 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
|
||||
}
|
||||
return err
|
||||
}
|
||||
defer save(name+".bkp", 30*time.Second) // Keep a backup as well
|
||||
defer save(name+".bkp", 5*time.Second) // Keep a backup as well
|
||||
|
||||
// drive timeout by default is 2 minutes, we do not need to wait longer.
|
||||
return save(name, time.Minute)
|
||||
|
||||
+8
-21
@@ -107,35 +107,22 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
|
||||
return nil
|
||||
}
|
||||
|
||||
cfgs := globalTierConfigMgr.ListTiers()
|
||||
if len(cfgs) == 0 {
|
||||
if globalTierConfigMgr.Empty() {
|
||||
return nil
|
||||
}
|
||||
|
||||
ts := make(map[string]madmin.TierStats, len(cfgs)+1)
|
||||
ts := make(map[string]madmin.TierStats)
|
||||
dui.TierStats.populateStats(ts)
|
||||
|
||||
infos := make([]madmin.TierInfo, 0, len(ts))
|
||||
|
||||
// Add STANDARD (hot-tier)
|
||||
ts[minioHotTier] = madmin.TierStats{}
|
||||
infos = append(infos, madmin.TierInfo{
|
||||
Name: minioHotTier,
|
||||
Type: "internal",
|
||||
})
|
||||
// Add configured remote tiers
|
||||
for _, cfg := range cfgs {
|
||||
ts[cfg.Name] = madmin.TierStats{}
|
||||
for tier, stats := range ts {
|
||||
infos = append(infos, madmin.TierInfo{
|
||||
Name: cfg.Name,
|
||||
Type: cfg.Type.String(),
|
||||
Name: tier,
|
||||
Type: globalTierConfigMgr.TierType(tier),
|
||||
Stats: stats,
|
||||
})
|
||||
}
|
||||
|
||||
ts = dui.TierStats.adminStats(ts)
|
||||
for i := range infos {
|
||||
info := infos[i]
|
||||
infos[i].Stats = ts[info.Name]
|
||||
}
|
||||
|
||||
sort.Slice(infos, func(i, j int) bool {
|
||||
if infos[i].Type == "internal" {
|
||||
return true
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,60 +0,0 @@
|
||||
//go:build windows
|
||||
// +build windows
|
||||
|
||||
/*
|
||||
* MinIO Object Storage (c) 2021 MinIO, Inc.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"github.com/djherbis/atime"
|
||||
"golang.org/x/sys/windows/registry"
|
||||
)
|
||||
|
||||
// Return error if Atime is disabled on the O/S
|
||||
func checkAtimeSupport(dir string) (err error) {
|
||||
file, err := os.CreateTemp(dir, "prefix")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer os.Remove(file.Name())
|
||||
defer file.Close()
|
||||
finfo1, err := os.Stat(file.Name())
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
atime.Get(finfo1)
|
||||
|
||||
k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Control\FileSystem`, registry.QUERY_VALUE)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer k.Close()
|
||||
|
||||
setting, _, err := k.GetIntegerValue("NtfsDisableLastAccessUpdate")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
lowSetting := setting & 0xFFFF
|
||||
if lowSetting != uint64(0x0000) && lowSetting != uint64(0x0002) {
|
||||
return errors.New("Atime not supported")
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// CacheDiskStats represents cache disk statistics
|
||||
// such as current disk usage and available.
|
||||
type CacheDiskStats struct {
|
||||
// used cache size
|
||||
UsageSize uint64
|
||||
// total cache disk capacity
|
||||
TotalCapacity uint64
|
||||
// indicates if usage is high or low, if high value is '1', if low its '0'
|
||||
UsageState int32
|
||||
// indicates the current usage percentage of this cache disk
|
||||
UsagePercent uint64
|
||||
Dir string
|
||||
}
|
||||
|
||||
// GetUsageLevelString gets the string representation for the usage level.
|
||||
func (c *CacheDiskStats) GetUsageLevelString() (u string) {
|
||||
if atomic.LoadInt32(&c.UsageState) == 0 {
|
||||
return "low"
|
||||
}
|
||||
return "high"
|
||||
}
|
||||
|
||||
// CacheStats - represents bytes served from cache,
|
||||
// cache hits and cache misses.
|
||||
type CacheStats struct {
|
||||
BytesServed uint64
|
||||
Hits uint64
|
||||
Misses uint64
|
||||
GetDiskStats func() []CacheDiskStats
|
||||
}
|
||||
|
||||
// Increase total bytes served from cache
|
||||
func (s *CacheStats) incBytesServed(n int64) {
|
||||
atomic.AddUint64(&s.BytesServed, uint64(n))
|
||||
}
|
||||
|
||||
// Increase cache hit by 1
|
||||
func (s *CacheStats) incHit() {
|
||||
atomic.AddUint64(&s.Hits, 1)
|
||||
}
|
||||
|
||||
// Increase cache miss by 1
|
||||
func (s *CacheStats) incMiss() {
|
||||
atomic.AddUint64(&s.Misses, 1)
|
||||
}
|
||||
|
||||
// Get total bytes served
|
||||
func (s *CacheStats) getBytesServed() uint64 {
|
||||
return atomic.LoadUint64(&s.BytesServed)
|
||||
}
|
||||
|
||||
// Get total cache hits
|
||||
func (s *CacheStats) getHits() uint64 {
|
||||
return atomic.LoadUint64(&s.Hits)
|
||||
}
|
||||
|
||||
// Get total cache misses
|
||||
func (s *CacheStats) getMisses() uint64 {
|
||||
return atomic.LoadUint64(&s.Misses)
|
||||
}
|
||||
|
||||
// Prepare new CacheStats structure
|
||||
func newCacheStats() *CacheStats {
|
||||
return &CacheStats{}
|
||||
}
|
||||
@@ -1,587 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"container/list"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/etag"
|
||||
)
|
||||
|
||||
// CacheStatusType - whether the request was served from cache.
|
||||
type CacheStatusType string
|
||||
|
||||
const (
|
||||
// CacheHit - whether object was served from cache.
|
||||
CacheHit CacheStatusType = "HIT"
|
||||
|
||||
// CacheMiss - object served from backend.
|
||||
CacheMiss CacheStatusType = "MISS"
|
||||
)
|
||||
|
||||
func (c CacheStatusType) String() string {
|
||||
if c != "" {
|
||||
return string(c)
|
||||
}
|
||||
return string(CacheMiss)
|
||||
}
|
||||
|
||||
type cacheControl struct {
|
||||
expiry time.Time
|
||||
maxAge int
|
||||
sMaxAge int
|
||||
minFresh int
|
||||
maxStale int
|
||||
noStore bool
|
||||
onlyIfCached bool
|
||||
noCache bool
|
||||
}
|
||||
|
||||
func (c *cacheControl) isStale(modTime time.Time) bool {
|
||||
if c == nil {
|
||||
return false
|
||||
}
|
||||
// response will never be stale if only-if-cached is set
|
||||
if c.onlyIfCached {
|
||||
return false
|
||||
}
|
||||
// Cache-Control value no-store indicates never cache
|
||||
if c.noStore {
|
||||
return true
|
||||
}
|
||||
// Cache-Control value no-cache indicates cache entry needs to be revalidated before
|
||||
// serving from cache
|
||||
if c.noCache {
|
||||
return true
|
||||
}
|
||||
now := time.Now()
|
||||
|
||||
if c.sMaxAge > 0 && c.sMaxAge < int(now.Sub(modTime).Seconds()) {
|
||||
return true
|
||||
}
|
||||
if c.maxAge > 0 && c.maxAge < int(now.Sub(modTime).Seconds()) {
|
||||
return true
|
||||
}
|
||||
|
||||
if !c.expiry.Equal(time.Time{}) && c.expiry.Before(time.Now().Add(time.Duration(c.maxStale))) {
|
||||
return true
|
||||
}
|
||||
|
||||
if c.minFresh > 0 && c.minFresh <= int(now.Sub(modTime).Seconds()) {
|
||||
return true
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
|
||||
// returns struct with cache-control settings from user metadata.
|
||||
func cacheControlOpts(o ObjectInfo) *cacheControl {
|
||||
c := cacheControl{}
|
||||
m := o.UserDefined
|
||||
if !o.Expires.Equal(timeSentinel) {
|
||||
c.expiry = o.Expires
|
||||
}
|
||||
|
||||
var headerVal string
|
||||
for k, v := range m {
|
||||
if strings.EqualFold(k, "cache-control") {
|
||||
headerVal = v
|
||||
}
|
||||
}
|
||||
if headerVal == "" {
|
||||
return nil
|
||||
}
|
||||
headerVal = strings.ToLower(headerVal)
|
||||
headerVal = strings.TrimSpace(headerVal)
|
||||
|
||||
vals := strings.Split(headerVal, ",")
|
||||
for _, val := range vals {
|
||||
val = strings.TrimSpace(val)
|
||||
|
||||
if val == "no-store" {
|
||||
c.noStore = true
|
||||
continue
|
||||
}
|
||||
if val == "only-if-cached" {
|
||||
c.onlyIfCached = true
|
||||
continue
|
||||
}
|
||||
if val == "no-cache" {
|
||||
c.noCache = true
|
||||
continue
|
||||
}
|
||||
p := strings.Split(val, "=")
|
||||
|
||||
if len(p) != 2 {
|
||||
continue
|
||||
}
|
||||
if p[0] == "max-age" ||
|
||||
p[0] == "s-maxage" ||
|
||||
p[0] == "min-fresh" ||
|
||||
p[0] == "max-stale" {
|
||||
i, err := strconv.Atoi(p[1])
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
if p[0] == "max-age" {
|
||||
c.maxAge = i
|
||||
}
|
||||
if p[0] == "s-maxage" {
|
||||
c.sMaxAge = i
|
||||
}
|
||||
if p[0] == "min-fresh" {
|
||||
c.minFresh = i
|
||||
}
|
||||
if p[0] == "max-stale" {
|
||||
c.maxStale = i
|
||||
}
|
||||
}
|
||||
}
|
||||
return &c
|
||||
}
|
||||
|
||||
// backendDownError returns true if err is due to backend failure or faulty disk if in server mode
|
||||
func backendDownError(err error) bool {
|
||||
_, backendDown := err.(BackendDown)
|
||||
return backendDown || IsErr(err, baseErrs...)
|
||||
}
|
||||
|
||||
// IsCacheable returns if the object should be saved in the cache.
|
||||
func (o ObjectInfo) IsCacheable() bool {
|
||||
if globalCacheKMS != nil {
|
||||
return true
|
||||
}
|
||||
_, ok := crypto.IsEncrypted(o.UserDefined)
|
||||
return !ok
|
||||
}
|
||||
|
||||
// reads file cached on disk from offset upto length
|
||||
func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser, error) {
|
||||
if filePath == "" || offset < 0 {
|
||||
return nil, errInvalidArgument
|
||||
}
|
||||
if err := checkPathLength(filePath); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
fr, err := os.Open(filePath)
|
||||
if err != nil {
|
||||
return nil, osErrToFileErr(err)
|
||||
}
|
||||
// Stat to get the size of the file at path.
|
||||
st, err := fr.Stat()
|
||||
if err != nil {
|
||||
err = osErrToFileErr(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Verify if its not a regular file, since subsequent Seek is undefined.
|
||||
if !st.Mode().IsRegular() {
|
||||
return nil, errIsNotRegular
|
||||
}
|
||||
|
||||
if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Seek to the requested offset.
|
||||
if offset > 0 {
|
||||
_, err = fr.Seek(offset, io.SeekStart)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return struct {
|
||||
io.Reader
|
||||
io.Closer
|
||||
}{Reader: io.LimitReader(fr, length), Closer: fr}, nil
|
||||
}
|
||||
|
||||
func isCacheEncrypted(meta map[string]string) bool {
|
||||
_, ok := meta[SSECacheEncrypted]
|
||||
return ok
|
||||
}
|
||||
|
||||
// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
|
||||
func decryptCacheObjectETag(info *ObjectInfo) error {
|
||||
if info.IsDir {
|
||||
return nil // Directories are never encrypted.
|
||||
}
|
||||
|
||||
// Depending on the SSE type we handle ETags slightly
|
||||
// differently. ETags encrypted with SSE-S3 must be
|
||||
// decrypted first, since the client expects that
|
||||
// a single-part SSE-S3 ETag is equal to the content MD5.
|
||||
//
|
||||
// For all other SSE types, the ETag is not the content MD5.
|
||||
// Therefore, we don't decrypt but only format it.
|
||||
switch kind, ok := crypto.IsEncrypted(info.UserDefined); {
|
||||
case ok && kind == crypto.S3 && isCacheEncrypted(info.UserDefined):
|
||||
ETag, err := etag.Parse(info.ETag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !ETag.IsEncrypted() {
|
||||
info.ETag = ETag.Format().String()
|
||||
return nil
|
||||
}
|
||||
|
||||
key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, info.UserDefined, info.Bucket, info.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ETag, err = etag.Decrypt(key[:], ETag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info.ETag = ETag.Format().String()
|
||||
case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(info.UserDefined):
|
||||
ETag, err := etag.Parse(info.ETag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
info.ETag = ETag.Format().String()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
|
||||
func decryptCachePartETags(c *cacheMeta) ([]string, error) {
|
||||
// Depending on the SSE type we handle ETags slightly
|
||||
// differently. ETags encrypted with SSE-S3 must be
|
||||
// decrypted first, since the client expects that
|
||||
// a single-part SSE-S3 ETag is equal to the content MD5.
|
||||
//
|
||||
// For all other SSE types, the ETag is not the content MD5.
|
||||
// Therefore, we don't decrypt but only format it.
|
||||
switch kind, ok := crypto.IsEncrypted(c.Meta); {
|
||||
case ok && kind == crypto.S3 && isCacheEncrypted(c.Meta):
|
||||
key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, c.Meta, c.Bucket, c.Object)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
etags := make([]string, 0, len(c.PartETags))
|
||||
for i := range c.PartETags {
|
||||
ETag, err := etag.Parse(c.PartETags[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ETag, err = etag.Decrypt(key[:], ETag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
etags = append(etags, ETag.Format().String())
|
||||
}
|
||||
return etags, nil
|
||||
case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(c.Meta):
|
||||
etags := make([]string, 0, len(c.PartETags))
|
||||
for i := range c.PartETags {
|
||||
ETag, err := etag.Parse(c.PartETags[i])
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
etags = append(etags, ETag.Format().String())
|
||||
}
|
||||
return etags, nil
|
||||
default:
|
||||
return c.PartETags, nil
|
||||
}
|
||||
}
|
||||
|
||||
func isMetadataSame(m1, m2 map[string]string) bool {
|
||||
if m1 == nil && m2 == nil {
|
||||
return true
|
||||
}
|
||||
if (m1 == nil && m2 != nil) || (m2 == nil && m1 != nil) {
|
||||
return false
|
||||
}
|
||||
if len(m1) != len(m2) {
|
||||
return false
|
||||
}
|
||||
for k1, v1 := range m1 {
|
||||
if v2, ok := m2[k1]; !ok || (v1 != v2) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
type fileScorer struct {
|
||||
saveBytes uint64
|
||||
now int64
|
||||
maxHits int
|
||||
// 1/size for consistent score.
|
||||
sizeMult float64
|
||||
|
||||
// queue is a linked list of files we want to delete.
|
||||
// The list is kept sorted according to score, highest at top, lowest at bottom.
|
||||
queue list.List
|
||||
queuedBytes uint64
|
||||
seenBytes uint64
|
||||
}
|
||||
|
||||
type queuedFile struct {
|
||||
name string
|
||||
versionID string
|
||||
size uint64
|
||||
score float64
|
||||
}
|
||||
|
||||
// newFileScorer allows to collect files to save a specific number of bytes.
|
||||
// Each file is assigned a score based on its age, size and number of hits.
|
||||
// A list of files is maintained
|
||||
func newFileScorer(saveBytes uint64, now int64, maxHits int) (*fileScorer, error) {
|
||||
if saveBytes == 0 {
|
||||
return nil, errors.New("newFileScorer: saveBytes = 0")
|
||||
}
|
||||
if now < 0 {
|
||||
return nil, errors.New("newFileScorer: now < 0")
|
||||
}
|
||||
if maxHits <= 0 {
|
||||
return nil, errors.New("newFileScorer: maxHits <= 0")
|
||||
}
|
||||
f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
|
||||
f.queue.Init()
|
||||
return &f, nil
|
||||
}
|
||||
|
||||
func (f *fileScorer) addFile(name string, accTime time.Time, size int64, hits int) {
|
||||
f.addFileWithObjInfo(ObjectInfo{
|
||||
Name: name,
|
||||
AccTime: accTime,
|
||||
Size: size,
|
||||
}, hits)
|
||||
}
|
||||
|
||||
func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
|
||||
// Calculate how much we want to delete this object.
|
||||
file := queuedFile{
|
||||
name: objInfo.Name,
|
||||
versionID: objInfo.VersionID,
|
||||
size: uint64(objInfo.Size),
|
||||
}
|
||||
f.seenBytes += uint64(objInfo.Size)
|
||||
|
||||
var score float64
|
||||
if objInfo.ModTime.IsZero() {
|
||||
// Mod time is not available with disk cache use atime.
|
||||
score = float64(f.now - objInfo.AccTime.Unix())
|
||||
} else {
|
||||
// if not used mod time when mod time is available.
|
||||
score = float64(f.now - objInfo.ModTime.Unix())
|
||||
}
|
||||
|
||||
// Size as fraction of how much we want to save, 0->1.
|
||||
szWeight := math.Max(0, (math.Min(1, float64(file.size)*f.sizeMult)))
|
||||
// 0 at f.maxHits, 1 at 0.
|
||||
hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
|
||||
file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
|
||||
// If we still haven't saved enough, just add the file
|
||||
if f.queuedBytes < f.saveBytes {
|
||||
f.insertFile(file)
|
||||
f.trimQueue()
|
||||
return
|
||||
}
|
||||
// If we score less than the worst, don't insert.
|
||||
worstE := f.queue.Back()
|
||||
if worstE != nil && file.score < worstE.Value.(queuedFile).score {
|
||||
return
|
||||
}
|
||||
f.insertFile(file)
|
||||
f.trimQueue()
|
||||
}
|
||||
|
||||
// adjustSaveBytes allows to adjust the number of bytes to save.
|
||||
// This can be used to adjust the count on the fly.
|
||||
// Returns true if there still is a need to delete files (n+saveBytes >0),
|
||||
// false if no more bytes needs to be saved.
|
||||
func (f *fileScorer) adjustSaveBytes(n int64) bool {
|
||||
if f == nil {
|
||||
return false
|
||||
}
|
||||
if int64(f.saveBytes)+n <= 0 {
|
||||
f.saveBytes = 0
|
||||
f.trimQueue()
|
||||
return false
|
||||
}
|
||||
if n < 0 {
|
||||
f.saveBytes -= ^uint64(n - 1)
|
||||
} else {
|
||||
f.saveBytes += uint64(n)
|
||||
}
|
||||
if f.saveBytes == 0 {
|
||||
f.queue.Init()
|
||||
f.saveBytes = 0
|
||||
return false
|
||||
}
|
||||
if n < 0 {
|
||||
f.trimQueue()
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// insertFile will insert a file into the list, sorted by its score.
|
||||
func (f *fileScorer) insertFile(file queuedFile) {
|
||||
e := f.queue.Front()
|
||||
for e != nil {
|
||||
v := e.Value.(queuedFile)
|
||||
if v.score < file.score {
|
||||
break
|
||||
}
|
||||
e = e.Next()
|
||||
}
|
||||
f.queuedBytes += file.size
|
||||
// We reached the end.
|
||||
if e == nil {
|
||||
f.queue.PushBack(file)
|
||||
return
|
||||
}
|
||||
f.queue.InsertBefore(file, e)
|
||||
}
|
||||
|
||||
// trimQueue will trim the back of queue and still keep below wantSave.
|
||||
func (f *fileScorer) trimQueue() {
|
||||
for {
|
||||
e := f.queue.Back()
|
||||
if e == nil {
|
||||
return
|
||||
}
|
||||
v := e.Value.(queuedFile)
|
||||
if f.queuedBytes-v.size < f.saveBytes {
|
||||
return
|
||||
}
|
||||
f.queue.Remove(e)
|
||||
f.queuedBytes -= v.size
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
|
||||
e := f.queue.Front()
|
||||
for e != nil {
|
||||
p(e.Value.(queuedFile))
|
||||
e = e.Next()
|
||||
}
|
||||
}
|
||||
|
||||
// fileNames returns all queued file names.
|
||||
func (f *fileScorer) fileNames() []string {
|
||||
res := make([]string, 0, f.queue.Len())
|
||||
e := f.queue.Front()
|
||||
for e != nil {
|
||||
res = append(res, e.Value.(queuedFile).name)
|
||||
e = e.Next()
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func (f *fileScorer) reset() {
|
||||
f.queue.Init()
|
||||
f.queuedBytes = 0
|
||||
}
|
||||
|
||||
func (f *fileScorer) queueString() string {
|
||||
var res strings.Builder
|
||||
e := f.queue.Front()
|
||||
i := 0
|
||||
for e != nil {
|
||||
v := e.Value.(queuedFile)
|
||||
if i > 0 {
|
||||
res.WriteByte('\n')
|
||||
}
|
||||
res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
|
||||
i++
|
||||
e = e.Next()
|
||||
}
|
||||
return res.String()
|
||||
}
|
||||
|
||||
// bytesToClear() returns the number of bytes to clear to reach low watermark
|
||||
// w.r.t quota given disk total and free space, quota in % allocated to cache
|
||||
// and low watermark % w.r.t allowed quota.
|
||||
// If the high watermark hasn't been reached 0 will be returned.
|
||||
func bytesToClear(total, free int64, quotaPct, lowWatermark, highWatermark uint64) uint64 {
|
||||
used := total - free
|
||||
quotaAllowed := total * (int64)(quotaPct) / 100
|
||||
highWMUsage := total * (int64)(highWatermark*quotaPct) / (100 * 100)
|
||||
if used < highWMUsage {
|
||||
return 0
|
||||
}
|
||||
// Return bytes needed to reach low watermark.
|
||||
lowWMUsage := total * (int64)(lowWatermark*quotaPct) / (100 * 100)
|
||||
return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
|
||||
}
|
||||
|
||||
type multiWriter struct {
|
||||
backendWriter io.Writer
|
||||
cacheWriter *io.PipeWriter
|
||||
pipeClosed bool
|
||||
}
|
||||
|
||||
// multiWriter writes to backend and cache - if cache write
|
||||
// fails close the pipe, but continue writing to the backend
|
||||
func (t *multiWriter) Write(p []byte) (n int, err error) {
|
||||
n, err = t.backendWriter.Write(p)
|
||||
if err == nil && n != len(p) {
|
||||
err = io.ErrShortWrite
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
if !t.pipeClosed {
|
||||
t.cacheWriter.CloseWithError(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// ignore errors writing to cache
|
||||
if !t.pipeClosed {
|
||||
_, cerr := t.cacheWriter.Write(p)
|
||||
if cerr != nil {
|
||||
t.pipeClosed = true
|
||||
t.cacheWriter.CloseWithError(cerr)
|
||||
}
|
||||
}
|
||||
return len(p), nil
|
||||
}
|
||||
|
||||
func cacheMultiWriter(w1 io.Writer, w2 *io.PipeWriter) io.Writer {
|
||||
return &multiWriter{backendWriter: w1, cacheWriter: w2}
|
||||
}
|
||||
|
||||
// writebackInProgress returns true if writeback commit is not complete
|
||||
func writebackInProgress(m map[string]string) bool {
|
||||
if v, ok := m[writeBackStatusHeader]; ok {
|
||||
switch cacheCommitStatus(v) {
|
||||
case CommitPending, CommitFailed:
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,177 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"reflect"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestGetCacheControlOpts(t *testing.T) {
|
||||
expiry, _ := time.Parse(http.TimeFormat, "Wed, 21 Oct 2015 07:28:00 GMT")
|
||||
|
||||
testCases := []struct {
|
||||
cacheControlHeaderVal string
|
||||
expiryHeaderVal time.Time
|
||||
expectedCacheControl *cacheControl
|
||||
expectedErr bool
|
||||
}{
|
||||
{"", timeSentinel, nil, false},
|
||||
{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
|
||||
{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
|
||||
{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
|
||||
{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
|
||||
{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 0o7, 28, 0o0, 0o0, time.UTC)}, false},
|
||||
{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
|
||||
}
|
||||
|
||||
for _, testCase := range testCases {
|
||||
t.Run("", func(t *testing.T) {
|
||||
m := make(map[string]string)
|
||||
m["cache-control"] = testCase.cacheControlHeaderVal
|
||||
if !testCase.expiryHeaderVal.Equal(timeSentinel) {
|
||||
m["expires"] = testCase.expiryHeaderVal.String()
|
||||
}
|
||||
c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
|
||||
if testCase.expectedErr && (c != nil) {
|
||||
t.Errorf("expected err, got <nil>")
|
||||
}
|
||||
if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
|
||||
t.Errorf("expected %v, got %v", testCase.expectedCacheControl, c)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsMetadataSame(t *testing.T) {
|
||||
testCases := []struct {
|
||||
m1 map[string]string
|
||||
m2 map[string]string
|
||||
expected bool
|
||||
}{
|
||||
{nil, nil, true},
|
||||
{nil, map[string]string{}, false},
|
||||
{map[string]string{"k": "v"}, map[string]string{"k": "v"}, true},
|
||||
{map[string]string{"k": "v"}, map[string]string{"a": "b"}, false},
|
||||
{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v1"}, false},
|
||||
{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, true},
|
||||
{map[string]string{"K1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
|
||||
{map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
actual := isMetadataSame(testCase.m1, testCase.m2)
|
||||
if testCase.expected != actual {
|
||||
t.Errorf("test %d expected %v, got %v", i, testCase.expected, actual)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewFileScorer(t *testing.T) {
|
||||
fs, err := newFileScorer(1000, time.Now().Unix(), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(fs.fileNames()) != 0 {
|
||||
t.Fatal("non zero files??")
|
||||
}
|
||||
now := time.Now()
|
||||
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
|
||||
fs.addFile("older", now.Add(-time.Hour), 1000, 10)
|
||||
if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
|
||||
t.Fatal("unexpected file list", fs.queueString())
|
||||
}
|
||||
fs.reset()
|
||||
fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
|
||||
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
|
||||
if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
|
||||
t.Fatal("unexpected file list", fs.queueString())
|
||||
}
|
||||
fs.reset()
|
||||
fs.addFile("less", now.Add(-time.Minute), 1000, 5)
|
||||
fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
|
||||
if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
|
||||
t.Fatal("unexpected file list", fs.queueString())
|
||||
}
|
||||
fs.reset()
|
||||
fs.addFile("small", now.Add(-time.Minute), 200, 10)
|
||||
fs.addFile("medium", now.Add(-time.Minute), 300, 10)
|
||||
if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
|
||||
t.Fatal("unexpected file list", fs.queueString())
|
||||
}
|
||||
fs.addFile("large", now.Add(-time.Minute), 700, 10)
|
||||
fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
|
||||
if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
|
||||
t.Fatal("unexpected file list", fs.queueString())
|
||||
}
|
||||
|
||||
fs.reset()
|
||||
fs.addFile("less", now.Add(-time.Minute), 500, 5)
|
||||
fs.addFile("recent", now.Add(-time.Minute), 500, 10)
|
||||
if !fs.adjustSaveBytes(-500) {
|
||||
t.Fatal("we should still need more bytes, got false")
|
||||
}
|
||||
// We should only need 500 bytes now.
|
||||
if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
|
||||
t.Fatal("unexpected file list", fs.queueString())
|
||||
}
|
||||
if fs.adjustSaveBytes(-500) {
|
||||
t.Fatal("we shouldn't need any more bytes, got true")
|
||||
}
|
||||
fs, err = newFileScorer(1000, time.Now().Unix(), 10)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
|
||||
// sorting should be consistent after adjusting savebytes.
|
||||
fs.adjustSaveBytes(-800)
|
||||
fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
|
||||
if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
|
||||
t.Fatal("unexpected file list", fs.queueString())
|
||||
}
|
||||
}
|
||||
|
||||
func TestBytesToClear(t *testing.T) {
|
||||
testCases := []struct {
|
||||
total int64
|
||||
free int64
|
||||
quotaPct uint64
|
||||
watermarkLow uint64
|
||||
watermarkHigh uint64
|
||||
expected uint64
|
||||
}{
|
||||
{total: 1000, free: 800, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 0},
|
||||
{total: 1000, free: 200, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 400},
|
||||
{total: 1000, free: 400, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 240},
|
||||
{total: 1000, free: 600, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 40},
|
||||
{total: 1000, free: 600, quotaPct: 40, watermarkLow: 70, watermarkHigh: 70, expected: 120},
|
||||
{total: 1000, free: 1000, quotaPct: 90, watermarkLow: 70, watermarkHigh: 70, expected: 0},
|
||||
|
||||
// High not yet reached..
|
||||
{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
|
||||
{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
|
||||
}
|
||||
for i, tc := range testCases {
|
||||
toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow, tc.watermarkHigh)
|
||||
if tc.expected != toClear {
|
||||
t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
|
||||
}
|
||||
}
|
||||
}
|
||||
-1218
File diff suppressed because it is too large
Load Diff
@@ -1,74 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Tests ToObjectInfo function.
|
||||
func TestCacheMetadataObjInfo(t *testing.T) {
|
||||
m := cacheMeta{Meta: nil}
|
||||
objInfo := m.ToObjectInfo()
|
||||
if objInfo.Size != 0 {
|
||||
t.Fatal("Unexpected object info value for Size", objInfo.Size)
|
||||
}
|
||||
if !objInfo.ModTime.Equal(time.Time{}) {
|
||||
t.Fatal("Unexpected object info value for ModTime ", objInfo.ModTime)
|
||||
}
|
||||
if objInfo.IsDir {
|
||||
t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
|
||||
}
|
||||
if !objInfo.Expires.IsZero() {
|
||||
t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
|
||||
}
|
||||
}
|
||||
|
||||
// test wildcard patterns for excluding entries from cache
|
||||
func TestCacheExclusion(t *testing.T) {
|
||||
cobjects := &cacheObjects{
|
||||
cache: nil,
|
||||
}
|
||||
|
||||
testCases := []struct {
|
||||
bucketName string
|
||||
objectName string
|
||||
excludePattern string
|
||||
expectedResult bool
|
||||
}{
|
||||
{"testbucket", "testobjectmatch", "testbucket/testobj*", true},
|
||||
{"testbucket", "testobjectnomatch", "testbucet/testobject*", false},
|
||||
{"testbucket", "testobject/pref1/obj1", "*/*", true},
|
||||
{"testbucket", "testobject/pref1/obj1", "*/pref1/*", true},
|
||||
{"testbucket", "testobject/pref1/obj1", "testobject/*", false},
|
||||
{"photos", "image1.jpg", "*.jpg", true},
|
||||
{"photos", "europe/paris/seine.jpg", "seine.jpg", false},
|
||||
{"photos", "europe/paris/seine.jpg", "*/seine.jpg", true},
|
||||
{"phil", "z/likes/coffee", "*/likes/*", true},
|
||||
{"failbucket", "no/slash/prefixes", "/failbucket/no/", false},
|
||||
{"failbucket", "no/slash/prefixes", "/failbucket/no/*", false},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
cobjects.exclude = []string{testCase.excludePattern}
|
||||
if cobjects.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
|
||||
t.Fatal("Cache exclusion test failed for case ", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -478,10 +478,7 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
|
||||
func decryptObjectMeta(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
|
||||
switch kind, _ := crypto.IsEncrypted(metadata); kind {
|
||||
case crypto.S3:
|
||||
var KMS kms.KMS = GlobalKMS
|
||||
if isCacheEncrypted(metadata) {
|
||||
KMS = globalCacheKMS
|
||||
}
|
||||
KMS := GlobalKMS
|
||||
if KMS == nil {
|
||||
return nil, errKMSNotConfigured
|
||||
}
|
||||
|
||||
+47
-43
@@ -915,33 +915,35 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
|
||||
}
|
||||
|
||||
orchestrated := IsKubernetes() || IsDocker()
|
||||
if !orchestrated {
|
||||
reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
|
||||
// If not orchestrated
|
||||
if !orchestrated &&
|
||||
// and not setup in reverse proxy
|
||||
!reverseProxy {
|
||||
// Check whether same path is not used in endpoints of a host on different port.
|
||||
// Only verify this on baremetal setups, DNS is not available in orchestrated
|
||||
// environments so we can't do much here.
|
||||
{
|
||||
pathIPMap := make(map[string]set.StringSet)
|
||||
hostIPCache := make(map[string]set.StringSet)
|
||||
for _, endpoint := range endpoints {
|
||||
host := endpoint.Hostname()
|
||||
hostIPSet, ok := hostIPCache[host]
|
||||
if !ok {
|
||||
var err error
|
||||
hostIPSet, err = getHostIP(host)
|
||||
if err != nil {
|
||||
return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
|
||||
}
|
||||
hostIPCache[host] = hostIPSet
|
||||
pathIPMap := make(map[string]set.StringSet)
|
||||
hostIPCache := make(map[string]set.StringSet)
|
||||
for _, endpoint := range endpoints {
|
||||
host := endpoint.Hostname()
|
||||
hostIPSet, ok := hostIPCache[host]
|
||||
if !ok {
|
||||
var err error
|
||||
hostIPSet, err = getHostIP(host)
|
||||
if err != nil {
|
||||
return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
|
||||
}
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return nil, setupType,
|
||||
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
|
||||
}
|
||||
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
|
||||
} else {
|
||||
pathIPMap[endpoint.Path] = hostIPSet
|
||||
hostIPCache[host] = hostIPSet
|
||||
}
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return nil, setupType,
|
||||
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
|
||||
}
|
||||
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
|
||||
} else {
|
||||
pathIPMap[endpoint.Path] = hostIPSet
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1113,32 +1115,34 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
|
||||
}
|
||||
|
||||
orchestrated := IsKubernetes() || IsDocker()
|
||||
if !orchestrated {
|
||||
reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
|
||||
// If not orchestrated
|
||||
if !orchestrated &&
|
||||
// and not setup in reverse proxy
|
||||
!reverseProxy {
|
||||
// Check whether same path is not used in endpoints of a host on different port.
|
||||
// Only verify this on baremetal setups, DNS is not available in orchestrated
|
||||
// environments so we can't do much here.
|
||||
{
|
||||
pathIPMap := make(map[string]set.StringSet)
|
||||
hostIPCache := make(map[string]set.StringSet)
|
||||
for _, endpoint := range endpoints {
|
||||
host := endpoint.Hostname()
|
||||
hostIPSet, ok := hostIPCache[host]
|
||||
if !ok {
|
||||
hostIPSet, err = getHostIP(host)
|
||||
if err != nil {
|
||||
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
|
||||
}
|
||||
hostIPCache[host] = hostIPSet
|
||||
pathIPMap := make(map[string]set.StringSet)
|
||||
hostIPCache := make(map[string]set.StringSet)
|
||||
for _, endpoint := range endpoints {
|
||||
host := endpoint.Hostname()
|
||||
hostIPSet, ok := hostIPCache[host]
|
||||
if !ok {
|
||||
hostIPSet, err = getHostIP(host)
|
||||
if err != nil {
|
||||
return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
|
||||
}
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return endpoints, setupType,
|
||||
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
|
||||
}
|
||||
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
|
||||
} else {
|
||||
pathIPMap[endpoint.Path] = hostIPSet
|
||||
hostIPCache[host] = hostIPSet
|
||||
}
|
||||
if IPSet, ok := pathIPMap[endpoint.Path]; ok {
|
||||
if !IPSet.Intersection(hostIPSet).IsEmpty() {
|
||||
return endpoints, setupType,
|
||||
config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
|
||||
}
|
||||
pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
|
||||
} else {
|
||||
pathIPMap[endpoint.Path] = hostIPSet
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+16
-1
@@ -62,7 +62,22 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
|
||||
return newDisks
|
||||
}
|
||||
|
||||
func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
|
||||
func (er erasureObjects) getOnlineLocalDisks() (newDisks []StorageAPI) {
|
||||
disks := er.getOnlineDisks()
|
||||
|
||||
// Based on the random shuffling return back randomized disks.
|
||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
for _, i := range r.Perm(len(disks)) {
|
||||
if disks[i] != nil && disks[i].IsLocal() {
|
||||
newDisks = append(newDisks, disks[i])
|
||||
}
|
||||
}
|
||||
|
||||
return newDisks
|
||||
}
|
||||
|
||||
func (er erasureObjects) getLocalDisks() (newDisks []StorageAPI) {
|
||||
disks := er.getDisks()
|
||||
// Based on the random shuffling return back randomized disks.
|
||||
r := rand.New(rand.NewSource(time.Now().UnixNano()))
|
||||
|
||||
@@ -1142,7 +1142,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
|
||||
if reqInfo != nil {
|
||||
newReqInfo = logger.NewReqInfo(reqInfo.RemoteHost, reqInfo.UserAgent, reqInfo.DeploymentID, reqInfo.RequestID, reqInfo.API, bucket, object)
|
||||
} else {
|
||||
newReqInfo = logger.NewReqInfo("", "", globalDeploymentID, "", "Heal", bucket, object)
|
||||
newReqInfo = logger.NewReqInfo("", "", globalDeploymentID(), "", "Heal", bucket, object)
|
||||
}
|
||||
healCtx := logger.SetReqInfo(GlobalContext, newReqInfo)
|
||||
|
||||
|
||||
@@ -180,6 +180,11 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
|
||||
objInfo.ReplicationStatusInternal = fi.ReplicationState.ReplicationStatusInternal
|
||||
objInfo.VersionPurgeStatusInternal = fi.ReplicationState.VersionPurgeStatusInternal
|
||||
objInfo.ReplicationStatus = fi.ReplicationStatus()
|
||||
if objInfo.ReplicationStatus.Empty() { // overlay x-amx-replication-status if present for replicas
|
||||
if st, ok := fi.Metadata[xhttp.AmzBucketReplicationStatus]; ok && st == string(replication.Replica) {
|
||||
objInfo.ReplicationStatus = replication.StatusType(st)
|
||||
}
|
||||
}
|
||||
objInfo.VersionPurgeStatus = fi.VersionPurgeStatus()
|
||||
|
||||
objInfo.TransitionedObject = TransitionedObject{
|
||||
|
||||
+125
-107
@@ -40,7 +40,6 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/pkg/v2/mimedb"
|
||||
"github.com/minio/pkg/v2/sync/errgroup"
|
||||
uatomic "go.uber.org/atomic"
|
||||
)
|
||||
|
||||
func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
|
||||
@@ -163,7 +162,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
|
||||
func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
|
||||
// run multiple cleanup's local to this server.
|
||||
var wg sync.WaitGroup
|
||||
for _, disk := range er.getLoadBalancedLocalDisks() {
|
||||
for _, disk := range er.getLocalDisks() {
|
||||
if disk != nil {
|
||||
wg.Add(1)
|
||||
go func(disk StorageAPI) {
|
||||
@@ -269,13 +268,19 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
|
||||
|
||||
var uploadIDs []string
|
||||
var disk StorageAPI
|
||||
disks := er.getLoadBalancedLocalDisks()
|
||||
disks := er.getOnlineLocalDisks()
|
||||
if len(disks) == 0 {
|
||||
// using er.getLoadBalancedLocalDisks() has one side-affect where
|
||||
// using er.getOnlineLocalDisks() has one side-affect where
|
||||
// on a pooled setup all disks are remote, add a fallback
|
||||
disks = er.getOnlineDisks()
|
||||
}
|
||||
for _, disk = range disks {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
if !disk.IsOnline() {
|
||||
continue
|
||||
}
|
||||
uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
|
||||
if err != nil {
|
||||
if errors.Is(err, errDiskNotFound) {
|
||||
@@ -304,20 +309,21 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
|
||||
if populatedUploadIds.Contains(uploadID) {
|
||||
continue
|
||||
}
|
||||
fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "", false)
|
||||
if err != nil {
|
||||
if !IsErrIgnored(err, errFileNotFound, errDiskNotFound) {
|
||||
logger.LogIf(ctx, err)
|
||||
// If present, use time stored in ID.
|
||||
startTime := time.Now()
|
||||
if split := strings.Split(uploadID, "x"); len(split) == 2 {
|
||||
t, err := strconv.ParseInt(split[1], 10, 64)
|
||||
if err == nil {
|
||||
startTime = time.Unix(0, t)
|
||||
}
|
||||
// Ignore this invalid upload-id since we are listing here
|
||||
continue
|
||||
}
|
||||
populatedUploadIds.Add(uploadID)
|
||||
uploads = append(uploads, MultipartInfo{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
UploadID: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID, uploadID))),
|
||||
Initiated: fi.ModTime,
|
||||
UploadID: base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID(), uploadID))),
|
||||
Initiated: startTime,
|
||||
})
|
||||
populatedUploadIds.Add(uploadID)
|
||||
}
|
||||
|
||||
sort.Slice(uploads, func(i int, j int) bool {
|
||||
@@ -399,47 +405,31 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
|
||||
// If we have offline disks upgrade the number of erasure codes for this object.
|
||||
parityOrig := parityDrives
|
||||
|
||||
atomicParityDrives := uatomic.NewInt64(0)
|
||||
atomicOfflineDrives := uatomic.NewInt64(0)
|
||||
|
||||
// Start with current parityDrives
|
||||
atomicParityDrives.Store(int64(parityDrives))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var offlineDrives int
|
||||
for _, disk := range onlineDisks {
|
||||
if disk == nil {
|
||||
atomicParityDrives.Inc()
|
||||
atomicOfflineDrives.Inc()
|
||||
parityDrives++
|
||||
offlineDrives++
|
||||
continue
|
||||
}
|
||||
if !disk.IsOnline() {
|
||||
atomicParityDrives.Inc()
|
||||
atomicOfflineDrives.Inc()
|
||||
parityDrives++
|
||||
offlineDrives++
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(disk StorageAPI) {
|
||||
defer wg.Done()
|
||||
di, err := disk.DiskInfo(ctx, false)
|
||||
if err != nil || di.ID == "" {
|
||||
atomicOfflineDrives.Inc()
|
||||
atomicParityDrives.Inc()
|
||||
}
|
||||
}(disk)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if int(atomicOfflineDrives.Load()) >= (len(onlineDisks)+1)/2 {
|
||||
if offlineDrives >= (len(onlineDisks)+1)/2 {
|
||||
// if offline drives are more than 50% of the drives
|
||||
// we have no quorum, we shouldn't proceed just
|
||||
// fail at that point.
|
||||
return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
|
||||
}
|
||||
|
||||
parityDrives = int(atomicParityDrives.Load())
|
||||
if parityDrives >= len(onlineDisks)/2 {
|
||||
parityDrives = len(onlineDisks) / 2
|
||||
}
|
||||
|
||||
if parityOrig != parityDrives {
|
||||
userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
|
||||
}
|
||||
@@ -492,7 +482,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
|
||||
partsMetadata[index].Metadata = userDefined
|
||||
}
|
||||
uploadUUID := mustGetUUID()
|
||||
uploadID := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID, uploadUUID)))
|
||||
uploadID := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID(), uploadUUID)))
|
||||
uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID)
|
||||
|
||||
// Write updated `xl.meta` to all disks.
|
||||
@@ -646,7 +636,8 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
|
||||
// accommodate concurrent PutObjectPart requests
|
||||
|
||||
partSuffix := fmt.Sprintf("part.%d", partID)
|
||||
tmpPart := mustGetUUID()
|
||||
// Random UUID and timestamp for temporary part file.
|
||||
tmpPart := fmt.Sprintf("%sx%d", mustGetUUID(), time.Now().UnixNano())
|
||||
tmpPartPath := pathJoin(tmpPart, partSuffix)
|
||||
|
||||
// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
|
||||
@@ -833,46 +824,8 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
|
||||
return result, toObjectErr(err, bucket, object, uploadID)
|
||||
}
|
||||
|
||||
onlineDisks := er.getDisks()
|
||||
uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
|
||||
|
||||
if maxParts == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
if partNumberMarker < 0 {
|
||||
partNumberMarker = 0
|
||||
}
|
||||
|
||||
// Limit output to maxPartsList.
|
||||
if maxParts > maxPartsList-partNumberMarker {
|
||||
maxParts = maxPartsList - partNumberMarker
|
||||
}
|
||||
|
||||
// Read Part info for all parts
|
||||
partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
|
||||
req := ReadMultipleReq{
|
||||
Bucket: minioMetaMultipartBucket,
|
||||
Prefix: partPath,
|
||||
MaxSize: 1 << 20, // Each part should realistically not be > 1MiB.
|
||||
MaxResults: maxParts + 1,
|
||||
}
|
||||
|
||||
start := partNumberMarker + 1
|
||||
end := start + maxParts
|
||||
|
||||
// Parts are 1 based, so index 0 is part one, etc.
|
||||
for i := start; i <= end; i++ {
|
||||
req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
|
||||
}
|
||||
|
||||
writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
|
||||
|
||||
partInfoFiles, err := readMultipleFiles(ctx, onlineDisks, req, writeQuorum)
|
||||
if err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
// Populate the result stub.
|
||||
result.Bucket = bucket
|
||||
result.Object = object
|
||||
@@ -882,33 +835,96 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
|
||||
result.UserDefined = cloneMSS(fi.Metadata)
|
||||
result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]
|
||||
|
||||
// For empty number of parts or maxParts as zero, return right here.
|
||||
if len(partInfoFiles) == 0 || maxParts == 0 {
|
||||
if partNumberMarker < 0 {
|
||||
partNumberMarker = 0
|
||||
}
|
||||
|
||||
// Limit output to maxPartsList.
|
||||
if maxParts > maxPartsList-partNumberMarker {
|
||||
maxParts = maxPartsList - partNumberMarker
|
||||
}
|
||||
|
||||
if maxParts == 0 {
|
||||
return result, nil
|
||||
}
|
||||
|
||||
for i, part := range partInfoFiles {
|
||||
partN := i + partNumberMarker + 1
|
||||
if part.Error != "" || !part.Exists {
|
||||
// Read Part info for all parts
|
||||
partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
|
||||
req := ReadMultipleReq{
|
||||
Bucket: minioMetaMultipartBucket,
|
||||
Prefix: partPath,
|
||||
MaxSize: 1 << 20, // Each part should realistically not be > 1MiB.
|
||||
MaxResults: maxParts + 1,
|
||||
MetadataOnly: true,
|
||||
}
|
||||
|
||||
start := partNumberMarker + 1
|
||||
end := start + maxParts
|
||||
|
||||
// Parts are 1 based, so index 0 is part one, etc.
|
||||
for i := start; i <= end; i++ {
|
||||
req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
|
||||
}
|
||||
|
||||
var disk StorageAPI
|
||||
disks := er.getOnlineLocalDisks()
|
||||
if len(disks) == 0 {
|
||||
// using er.getOnlineLocalDisks() has one side-affect where
|
||||
// on a pooled setup all disks are remote, add a fallback
|
||||
disks = er.getOnlineDisks()
|
||||
}
|
||||
|
||||
for _, disk = range disks {
|
||||
if disk == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
var pfi FileInfo
|
||||
_, err := pfi.UnmarshalMsg(part.Data)
|
||||
if err != nil {
|
||||
// Maybe crash or similar.
|
||||
logger.LogIf(ctx, err)
|
||||
if !disk.IsOnline() {
|
||||
continue
|
||||
}
|
||||
|
||||
partI := pfi.Parts[0]
|
||||
if partN != partI.Number {
|
||||
logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", i+1, i+1, partI.Number))
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
|
||||
// Add the current part.
|
||||
fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index, partI.Checksums)
|
||||
g := errgroup.WithNErrs(len(req.Files)).WithConcurrency(32)
|
||||
|
||||
partsInfo := make([]ObjectPartInfo, len(req.Files))
|
||||
for i, file := range req.Files {
|
||||
file := file
|
||||
partN := i + start
|
||||
i := i
|
||||
|
||||
g.Go(func() error {
|
||||
buf, err := disk.ReadAll(ctx, minioMetaMultipartBucket, pathJoin(partPath, file))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var pfi FileInfo
|
||||
_, err = pfi.UnmarshalMsg(buf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(pfi.Parts) != 1 {
|
||||
return errors.New("invalid number of parts expected 1, got 0")
|
||||
}
|
||||
|
||||
if partN != pfi.Parts[0].Number {
|
||||
return fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partN, partN, pfi.Parts[0].Number)
|
||||
}
|
||||
|
||||
partsInfo[i] = pfi.Parts[0]
|
||||
return nil
|
||||
}, i)
|
||||
}
|
||||
|
||||
g.Wait()
|
||||
|
||||
for _, part := range partsInfo {
|
||||
if part.Number != 0 && !part.ModTime.IsZero() {
|
||||
fi.AddObjectPart(part.Number, part.ETag, part.Size, part.ActualSize, part.ModTime, part.Index, part.Checksums)
|
||||
}
|
||||
}
|
||||
|
||||
// Only parts with higher part numbers will be listed.
|
||||
@@ -973,11 +989,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
// Read Part info for all parts
|
||||
partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
|
||||
req := ReadMultipleReq{
|
||||
Bucket: minioMetaMultipartBucket,
|
||||
Prefix: partPath,
|
||||
MaxSize: 1 << 20, // Each part should realistically not be > 1MiB.
|
||||
Files: make([]string, 0, len(parts)),
|
||||
AbortOn404: true,
|
||||
Bucket: minioMetaMultipartBucket,
|
||||
Prefix: partPath,
|
||||
MaxSize: 1 << 20, // Each part should realistically not be > 1MiB.
|
||||
Files: make([]string, 0, len(parts)),
|
||||
AbortOn404: true,
|
||||
MetadataOnly: true,
|
||||
}
|
||||
for _, part := range parts {
|
||||
req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", part.PartNumber))
|
||||
@@ -1180,6 +1197,16 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
return oi, err
|
||||
}
|
||||
}
|
||||
|
||||
// Hold namespace to complete the transaction
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return oi, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
if checksumType.IsSet() {
|
||||
checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
|
||||
cs := hash.NewChecksumFromData(checksumType, checksumCombined)
|
||||
@@ -1223,15 +1250,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
|
||||
}
|
||||
}
|
||||
|
||||
// Hold namespace to complete the transaction
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
|
||||
if err != nil {
|
||||
return oi, err
|
||||
}
|
||||
ctx = lkctx.Context()
|
||||
defer lk.Unlock(lkctx)
|
||||
|
||||
// Remove parts that weren't present in CompleteMultipartUpload request.
|
||||
for _, curpart := range currentFI.Parts {
|
||||
// Remove part.meta which is not needed anymore.
|
||||
|
||||
+59
-45
@@ -47,7 +47,6 @@ import (
|
||||
"github.com/minio/pkg/v2/sync/errgroup"
|
||||
"github.com/minio/pkg/v2/wildcard"
|
||||
"github.com/tinylib/msgp/msgp"
|
||||
uatomic "go.uber.org/atomic"
|
||||
)
|
||||
|
||||
// list all errors which can be ignored in object operations.
|
||||
@@ -133,15 +132,18 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
||||
versionID = mustGetUUID()
|
||||
fi.IsLatest = true // we are creating a new version so this is latest.
|
||||
}
|
||||
modTime = UTCNow()
|
||||
}
|
||||
|
||||
modTime = UTCNow() // We only preserve modTime if dstOpts.MTime is true.
|
||||
// in all other cases mtime is latest.
|
||||
|
||||
fi.VersionID = versionID // set any new versionID we might have created
|
||||
fi.ModTime = modTime // set modTime for the new versionID
|
||||
if !dstOpts.MTime.IsZero() {
|
||||
modTime = dstOpts.MTime
|
||||
fi.ModTime = dstOpts.MTime
|
||||
}
|
||||
|
||||
fi.Metadata = srcInfo.UserDefined
|
||||
srcInfo.UserDefined["etag"] = srcInfo.ETag
|
||||
|
||||
@@ -189,14 +191,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
|
||||
func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
|
||||
auditObjectErasureSet(ctx, object, &er)
|
||||
|
||||
// This is a special call attempted first to check for SOS-API calls.
|
||||
gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts)
|
||||
if err == nil {
|
||||
return gr, nil
|
||||
}
|
||||
// reset any error to 'nil'
|
||||
err = nil
|
||||
|
||||
var unlockOnDefer bool
|
||||
nsUnlocker := func() {}
|
||||
defer func() {
|
||||
@@ -452,14 +446,6 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
|
||||
func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
|
||||
auditObjectErasureSet(ctx, object, &er)
|
||||
|
||||
// This is a special call attempted first to check for SOS-API calls.
|
||||
info, err = veeamSOSAPIHeadObject(ctx, bucket, object, opts)
|
||||
if err == nil {
|
||||
return info, nil
|
||||
}
|
||||
// reset any error to 'nil'
|
||||
err = nil
|
||||
|
||||
if !opts.NoLock {
|
||||
// Lock the object before reading.
|
||||
lk := er.NewNSLock(bucket, object)
|
||||
@@ -490,6 +476,24 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
|
||||
auditLogInternal(ctx, opts)
|
||||
}
|
||||
|
||||
func joinErrors(errs ...error) error {
|
||||
s := make([]string, 0, len(errs))
|
||||
nonNilErrs := make([]any, 0, len(errs))
|
||||
for _, err := range errs {
|
||||
if err == nil {
|
||||
continue
|
||||
}
|
||||
s = append(s, "[%w]")
|
||||
nonNilErrs = append(nonNilErrs, err)
|
||||
}
|
||||
// If all the errors were nil, return nil.
|
||||
if len(nonNilErrs) == 0 {
|
||||
return nil
|
||||
}
|
||||
allErrs := strings.Join(s, "\n")
|
||||
return fmt.Errorf(allErrs, nonNilErrs...)
|
||||
}
|
||||
|
||||
func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
|
||||
_, file, line, cok := runtime.Caller(1)
|
||||
var err error
|
||||
@@ -498,8 +502,8 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
|
||||
tags := make(map[string]interface{}, 4)
|
||||
tags["set"] = er.setIndex
|
||||
tags["pool"] = er.poolIndex
|
||||
tags["merrs"] = errors.Join(errs...)
|
||||
tags["derrs"] = errors.Join(dataErrs...)
|
||||
tags["merrs"] = joinErrors(errs...) // errors.Join(errs...)
|
||||
tags["derrs"] = joinErrors(dataErrs...) // errors.Join(dataErrs...)
|
||||
if m.IsValid() {
|
||||
tags["size"] = m.Size
|
||||
tags["mtime"] = m.ModTime.Format(http.TimeFormat)
|
||||
@@ -662,6 +666,32 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
|
||||
return metaFileInfos, errs
|
||||
}
|
||||
|
||||
// Checking if an object is dangling costs some IOPS; hence implementing this function
|
||||
// which decides which condition it is useful to check if an object is dangling
|
||||
//
|
||||
// errs: errors from reading xl.meta in all disks
|
||||
// err: reduced errs
|
||||
// bucket: the object name in question
|
||||
func shouldCheckForDangling(err error, errs []error, bucket string) bool {
|
||||
// Avoid data in .minio.sys for now
|
||||
if bucket == minioMetaBucket {
|
||||
return false
|
||||
}
|
||||
switch {
|
||||
// Check if we have a read quorum issue
|
||||
case errors.Is(err, errErasureReadQuorum):
|
||||
return true
|
||||
// Check if the object is inexistent in most disks but not all of them
|
||||
case errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound):
|
||||
for i := range errs {
|
||||
if errs[i] == nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI, err error) {
|
||||
disks := er.getDisks()
|
||||
|
||||
@@ -676,7 +706,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
||||
|
||||
readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
|
||||
if err != nil {
|
||||
if errors.Is(err, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
|
||||
if shouldCheckForDangling(err, errs, bucket) {
|
||||
_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
|
||||
if derr != nil {
|
||||
err = derr
|
||||
@@ -686,7 +716,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
|
||||
}
|
||||
|
||||
if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
|
||||
if errors.Is(reducedErr, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
|
||||
if shouldCheckForDangling(err, errs, bucket) {
|
||||
_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
|
||||
if derr != nil {
|
||||
reducedErr = derr
|
||||
@@ -1097,47 +1127,31 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
|
||||
// If we have offline disks upgrade the number of erasure codes for this object.
|
||||
parityOrig := parityDrives
|
||||
|
||||
atomicParityDrives := uatomic.NewInt64(0)
|
||||
atomicOfflineDrives := uatomic.NewInt64(0)
|
||||
|
||||
// Start with current parityDrives
|
||||
atomicParityDrives.Store(int64(parityDrives))
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var offlineDrives int
|
||||
for _, disk := range storageDisks {
|
||||
if disk == nil {
|
||||
atomicParityDrives.Inc()
|
||||
atomicOfflineDrives.Inc()
|
||||
parityDrives++
|
||||
offlineDrives++
|
||||
continue
|
||||
}
|
||||
if !disk.IsOnline() {
|
||||
atomicParityDrives.Inc()
|
||||
atomicOfflineDrives.Inc()
|
||||
parityDrives++
|
||||
offlineDrives++
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(disk StorageAPI) {
|
||||
defer wg.Done()
|
||||
di, err := disk.DiskInfo(ctx, false)
|
||||
if err != nil || di.ID == "" {
|
||||
atomicOfflineDrives.Inc()
|
||||
atomicParityDrives.Inc()
|
||||
}
|
||||
}(disk)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
if int(atomicOfflineDrives.Load()) >= (len(storageDisks)+1)/2 {
|
||||
if offlineDrives >= (len(storageDisks)+1)/2 {
|
||||
// if offline drives are more than 50% of the drives
|
||||
// we have no quorum, we shouldn't proceed just
|
||||
// fail at that point.
|
||||
return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
|
||||
}
|
||||
|
||||
parityDrives = int(atomicParityDrives.Load())
|
||||
if parityDrives >= len(storageDisks)/2 {
|
||||
parityDrives = len(storageDisks) / 2
|
||||
}
|
||||
|
||||
if parityOrig != parityDrives {
|
||||
userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
|
||||
}
|
||||
|
||||
@@ -160,6 +160,9 @@ func (ps PoolStatus) Clone() PoolStatus {
|
||||
type poolMeta struct {
|
||||
Version int `msg:"v"`
|
||||
Pools []PoolStatus `msg:"pls"`
|
||||
|
||||
// Value should not be saved when we have not loaded anything yet.
|
||||
dontSave bool `msg:"-"`
|
||||
}
|
||||
|
||||
// A decommission resumable tells us if decommission is worth
|
||||
@@ -308,6 +311,10 @@ func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
|
||||
}
|
||||
|
||||
func (p poolMeta) IsSuspended(idx int) bool {
|
||||
if idx >= len(p.Pools) {
|
||||
// We don't really know if the pool is suspended or not, since it doesn't exist.
|
||||
return false
|
||||
}
|
||||
return p.Pools[idx].Decommission != nil
|
||||
}
|
||||
|
||||
@@ -445,6 +452,9 @@ func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSet
|
||||
}
|
||||
|
||||
func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
|
||||
if p.dontSave {
|
||||
return nil
|
||||
}
|
||||
data := make([]byte, 4, p.Msgsize()+4)
|
||||
|
||||
// Initialize the header.
|
||||
@@ -499,36 +509,17 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
|
||||
|
||||
// if no update is needed return right away.
|
||||
if !update {
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta = meta
|
||||
z.poolMetaMutex.Unlock()
|
||||
} else {
|
||||
newMeta := poolMeta{} // to update write poolMeta fresh.
|
||||
// looks like new pool was added we need to update,
|
||||
// or this is a fresh installation (or an existing
|
||||
// installation with pool removed)
|
||||
newMeta.Version = poolMetaVersion
|
||||
for idx, pool := range z.serverPools {
|
||||
var skip bool
|
||||
for _, currentPool := range meta.Pools {
|
||||
// Preserve any current pool status.
|
||||
if currentPool.CmdLine == pool.endpoints.CmdLine {
|
||||
newMeta.Pools = append(newMeta.Pools, currentPool)
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
newMeta.Pools = append(newMeta.Pools, PoolStatus{
|
||||
CmdLine: pool.endpoints.CmdLine,
|
||||
ID: idx,
|
||||
LastUpdate: UTCNow(),
|
||||
})
|
||||
}
|
||||
newMeta := newPoolMeta(z, meta)
|
||||
if err = newMeta.save(ctx, z.serverPools); err != nil {
|
||||
return err
|
||||
}
|
||||
z.poolMetaMutex.Lock()
|
||||
z.poolMeta = newMeta
|
||||
z.poolMetaMutex.Unlock()
|
||||
}
|
||||
|
||||
pools := meta.returnResumablePools()
|
||||
@@ -568,6 +559,34 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func newPoolMeta(z *erasureServerPools, prevMeta poolMeta) poolMeta {
|
||||
newMeta := poolMeta{} // to update write poolMeta fresh.
|
||||
// looks like new pool was added we need to update,
|
||||
// or this is a fresh installation (or an existing
|
||||
// installation with pool removed)
|
||||
newMeta.Version = poolMetaVersion
|
||||
for idx, pool := range z.serverPools {
|
||||
var skip bool
|
||||
for _, currentPool := range prevMeta.Pools {
|
||||
// Preserve any current pool status.
|
||||
if currentPool.CmdLine == pool.endpoints.CmdLine {
|
||||
newMeta.Pools = append(newMeta.Pools, currentPool)
|
||||
skip = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if skip {
|
||||
continue
|
||||
}
|
||||
newMeta.Pools = append(newMeta.Pools, PoolStatus{
|
||||
CmdLine: pool.endpoints.CmdLine,
|
||||
ID: idx,
|
||||
LastUpdate: UTCNow(),
|
||||
})
|
||||
}
|
||||
return newMeta
|
||||
}
|
||||
|
||||
func (z *erasureServerPools) IsDecommissionRunning() bool {
|
||||
z.poolMetaMutex.RLock()
|
||||
defer z.poolMetaMutex.RUnlock()
|
||||
@@ -682,10 +701,12 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
|
||||
return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints())
|
||||
}
|
||||
|
||||
listQuorum := (len(disks) + 1) / 2
|
||||
|
||||
// How to resolve partial results.
|
||||
resolver := metadataResolutionParams{
|
||||
dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
|
||||
objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
|
||||
dirQuorum: listQuorum, // make sure to capture all quorum ratios
|
||||
objQuorum: listQuorum, // make sure to capture all quorum ratios
|
||||
bucket: bi.Name,
|
||||
}
|
||||
|
||||
@@ -695,7 +716,7 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
|
||||
path: bi.Prefix,
|
||||
recursive: true,
|
||||
forwardTo: "",
|
||||
minDisks: len(disks) / 2, // to capture all quorum ratios
|
||||
minDisks: len(disks) / 2,
|
||||
reportNotFound: false,
|
||||
agreed: fn,
|
||||
partial: func(entries metaCacheEntries, _ []error) {
|
||||
@@ -1003,8 +1024,15 @@ func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err
|
||||
|
||||
func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
|
||||
pool := z.serverPools[idx]
|
||||
for _, bucket := range z.poolMeta.PendingBuckets(idx) {
|
||||
if z.poolMeta.isBucketDecommissioned(idx, bucket.String()) {
|
||||
z.poolMetaMutex.RLock()
|
||||
pending := z.poolMeta.PendingBuckets(idx)
|
||||
z.poolMetaMutex.RUnlock()
|
||||
|
||||
for _, bucket := range pending {
|
||||
z.poolMetaMutex.RLock()
|
||||
isDecommissioned := z.poolMeta.isBucketDecommissioned(idx, bucket.String())
|
||||
z.poolMetaMutex.RUnlock()
|
||||
if isDecommissioned {
|
||||
if serverDebugLog {
|
||||
console.Debugln("decommission: already done, moving on", bucket)
|
||||
}
|
||||
|
||||
+51
-34
@@ -143,6 +143,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
|
||||
|
||||
z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
|
||||
|
||||
// Initialize the pool meta, but set it to not save.
|
||||
// When z.Init below has loaded the poolmeta will be initialized,
|
||||
// and allowed to save.
|
||||
z.poolMeta = newPoolMeta(z, poolMeta{})
|
||||
z.poolMeta.dontSave = true
|
||||
|
||||
// initialize the object layer.
|
||||
setObjectLayer(z)
|
||||
|
||||
@@ -789,6 +795,16 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// This is a special call attempted first to check for SOS-API calls.
|
||||
gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts)
|
||||
if err == nil {
|
||||
return gr, nil
|
||||
}
|
||||
|
||||
// reset any error to 'nil' and any reader to be 'nil'
|
||||
gr = nil
|
||||
err = nil
|
||||
|
||||
object = encodeDirObject(object)
|
||||
|
||||
if z.SinglePool() {
|
||||
@@ -918,6 +934,16 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
|
||||
return objInfo, err
|
||||
}
|
||||
|
||||
// This is a special call attempted first to check for SOS-API calls.
|
||||
objInfo, err = veeamSOSAPIHeadObject(ctx, bucket, object, opts)
|
||||
if err == nil {
|
||||
return objInfo, nil
|
||||
}
|
||||
|
||||
// reset any error to 'nil', and object info to be empty.
|
||||
err = nil
|
||||
objInfo = ObjectInfo{}
|
||||
|
||||
object = encodeDirObject(object)
|
||||
|
||||
if z.SinglePool() {
|
||||
@@ -944,7 +970,7 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
|
||||
// PutObject - writes an object to least used erasure pool.
|
||||
func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
|
||||
// Validate put object input args.
|
||||
if err := checkPutObjectArgs(ctx, bucket, object, z); err != nil {
|
||||
if err := checkPutObjectArgs(ctx, bucket, object); err != nil {
|
||||
return ObjectInfo{}, err
|
||||
}
|
||||
|
||||
@@ -2227,11 +2253,12 @@ type HealthOptions struct {
|
||||
// additionally with any specific heuristic information which
|
||||
// was queried
|
||||
type HealthResult struct {
|
||||
Healthy bool
|
||||
HealingDrives int
|
||||
UnhealthyPools []struct {
|
||||
Healthy bool
|
||||
HealingDrives int
|
||||
ESHealth []struct {
|
||||
Maintenance bool
|
||||
PoolID, SetID int
|
||||
HealthyDrives int
|
||||
WriteQuorum int
|
||||
}
|
||||
WriteQuorum int
|
||||
@@ -2346,50 +2373,40 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
|
||||
}
|
||||
|
||||
result := HealthResult{
|
||||
HealingDrives: len(aggHealStateResult.HealDisks),
|
||||
Healthy: true,
|
||||
WriteQuorum: maximumWriteQuorum,
|
||||
UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
|
||||
}
|
||||
|
||||
for poolIdx := range erasureSetUpCount {
|
||||
for setIdx := range erasureSetUpCount[poolIdx] {
|
||||
result.ESHealth = append(result.ESHealth, struct {
|
||||
Maintenance bool
|
||||
PoolID, SetID int
|
||||
HealthyDrives, WriteQuorum int
|
||||
}{
|
||||
Maintenance: opts.Maintenance,
|
||||
SetID: setIdx,
|
||||
PoolID: poolIdx,
|
||||
HealthyDrives: erasureSetUpCount[poolIdx][setIdx],
|
||||
WriteQuorum: poolWriteQuorums[poolIdx],
|
||||
})
|
||||
|
||||
if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
|
||||
logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
|
||||
fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
|
||||
poolIdx, setIdx, poolWriteQuorums[poolIdx]))
|
||||
result.UnhealthyPools = append(result.UnhealthyPools, struct {
|
||||
Maintenance bool
|
||||
PoolID, SetID, WriteQuorum int
|
||||
}{
|
||||
Maintenance: opts.Maintenance,
|
||||
SetID: setIdx,
|
||||
PoolID: poolIdx,
|
||||
WriteQuorum: poolWriteQuorums[poolIdx],
|
||||
})
|
||||
result.Healthy = false
|
||||
}
|
||||
}
|
||||
if len(result.UnhealthyPools) > 0 {
|
||||
// We have unhealthy pools return error.
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
// when maintenance is not specified we don't have
|
||||
// to look at the healing side of the code.
|
||||
if !opts.Maintenance {
|
||||
return HealthResult{
|
||||
Healthy: true,
|
||||
WriteQuorum: maximumWriteQuorum,
|
||||
UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
|
||||
}
|
||||
if opts.Maintenance {
|
||||
result.Healthy = result.Healthy && len(aggHealStateResult.HealDisks) == 0
|
||||
result.HealingDrives = len(aggHealStateResult.HealDisks)
|
||||
}
|
||||
|
||||
return HealthResult{
|
||||
Healthy: len(aggHealStateResult.HealDisks) == 0,
|
||||
HealingDrives: len(aggHealStateResult.HealDisks),
|
||||
WriteQuorum: maximumWriteQuorum,
|
||||
UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// PutObjectMetadata - replace or add tags to an existing object
|
||||
@@ -2451,12 +2468,12 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
|
||||
return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
|
||||
}
|
||||
|
||||
idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
|
||||
oi, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return z.serverPools[idx].GetObjectTags(ctx, bucket, object, opts)
|
||||
return tags.ParseObjectTags(oi.UserTags)
|
||||
}
|
||||
|
||||
// TransitionObject - transition object content to target tier.
|
||||
|
||||
@@ -217,6 +217,10 @@ func (s *erasureSets) connectDisks() {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if cdisk != nil {
|
||||
// Close previous offline disk.
|
||||
cdisk.Close()
|
||||
}
|
||||
|
||||
wg.Add(1)
|
||||
go func(endpoint Endpoint) {
|
||||
|
||||
@@ -18,7 +18,6 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
@@ -26,7 +25,6 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/klauspost/reedsolomon"
|
||||
xioutil "github.com/minio/minio/internal/ioutil"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
@@ -85,7 +83,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
|
||||
|
||||
// We have written all the blocks, write the last remaining block.
|
||||
if write < int64(len(block)) {
|
||||
n, err := xioutil.Copy(dst, bytes.NewReader(block[:write]))
|
||||
n, err := dst.Write(block[:write])
|
||||
if err != nil {
|
||||
// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
|
||||
// The reader pipe might be closed at ListObjects io.EOF ignore it.
|
||||
@@ -94,12 +92,12 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
totalWritten += n
|
||||
totalWritten += int64(n)
|
||||
break
|
||||
}
|
||||
|
||||
// Copy the block.
|
||||
n, err := xioutil.Copy(dst, bytes.NewReader(block))
|
||||
n, err := dst.Write(block)
|
||||
if err != nil {
|
||||
// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
|
||||
// The reader pipe might be closed at ListObjects io.EOF ignore it.
|
||||
@@ -110,10 +108,10 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
|
||||
}
|
||||
|
||||
// Decrement output size.
|
||||
write -= n
|
||||
write -= int64(n)
|
||||
|
||||
// Increment written.
|
||||
totalWritten += n
|
||||
totalWritten += int64(n)
|
||||
}
|
||||
|
||||
// Success.
|
||||
|
||||
+3
-2
@@ -335,7 +335,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
|
||||
func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
|
||||
// run multiple cleanup's local to this server.
|
||||
var wg sync.WaitGroup
|
||||
for _, disk := range er.getLoadBalancedLocalDisks() {
|
||||
for _, disk := range er.getLocalDisks() {
|
||||
if disk != nil {
|
||||
wg.Add(1)
|
||||
go func(disk StorageAPI) {
|
||||
@@ -433,6 +433,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
|
||||
}
|
||||
logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update")
|
||||
updates <- cache.clone()
|
||||
|
||||
lastSave = cache.Info.LastUpdate
|
||||
case v, ok := <-bucketResults:
|
||||
if !ok {
|
||||
@@ -440,7 +441,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
|
||||
cache.Info.NextCycle = wantCycle
|
||||
cache.Info.LastUpdate = time.Now()
|
||||
logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed")
|
||||
updates <- cache
|
||||
updates <- cache.clone()
|
||||
return
|
||||
}
|
||||
cache.replace(v.Name, v.Parent, v.Entry)
|
||||
|
||||
@@ -104,7 +104,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
|
||||
for res := range evnot.targetResCh {
|
||||
if res.Err != nil {
|
||||
reqInfo := &logger.ReqInfo{}
|
||||
reqInfo.AppendTags("targetID", res.ID.Name)
|
||||
reqInfo.AppendTags("targetID", res.ID.String())
|
||||
logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
|
||||
}
|
||||
}
|
||||
@@ -128,40 +128,6 @@ func (evnot *EventNotifier) AddRulesMap(bucketName string, rulesMap event.RulesM
|
||||
}
|
||||
}
|
||||
|
||||
// RemoveRulesMap - removes rules map for bucket name.
|
||||
func (evnot *EventNotifier) RemoveRulesMap(bucketName string, rulesMap event.RulesMap) {
|
||||
evnot.Lock()
|
||||
defer evnot.Unlock()
|
||||
|
||||
evnot.bucketRulesMap[bucketName].Remove(rulesMap)
|
||||
if len(evnot.bucketRulesMap[bucketName]) == 0 {
|
||||
delete(evnot.bucketRulesMap, bucketName)
|
||||
}
|
||||
}
|
||||
|
||||
// ConfiguredTargetIDs - returns list of configured target id's
|
||||
func (evnot *EventNotifier) ConfiguredTargetIDs() []event.TargetID {
|
||||
if evnot == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
evnot.RLock()
|
||||
defer evnot.RUnlock()
|
||||
|
||||
var targetIDs []event.TargetID
|
||||
for _, rmap := range evnot.bucketRulesMap {
|
||||
for _, rules := range rmap {
|
||||
for _, targetSet := range rules {
|
||||
for id := range targetSet {
|
||||
targetIDs = append(targetIDs, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return targetIDs
|
||||
}
|
||||
|
||||
// RemoveNotification - removes all notification configuration for bucket name.
|
||||
func (evnot *EventNotifier) RemoveNotification(bucketName string) {
|
||||
evnot.Lock()
|
||||
@@ -210,6 +176,9 @@ type eventArgs struct {
|
||||
func (args eventArgs) ToEvent(escape bool) event.Event {
|
||||
eventTime := UTCNow()
|
||||
uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
|
||||
if !args.Object.ModTime.IsZero() {
|
||||
uniqueID = fmt.Sprintf("%X", args.Object.ModTime.UnixNano())
|
||||
}
|
||||
|
||||
respElements := map[string]string{
|
||||
"x-amz-request-id": args.RespElements["requestId"],
|
||||
@@ -223,7 +192,7 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
|
||||
}
|
||||
|
||||
// Add deployment as part of response elements.
|
||||
respElements["x-minio-deployment-id"] = globalDeploymentID
|
||||
respElements["x-minio-deployment-id"] = globalDeploymentID()
|
||||
if args.RespElements["content-length"] != "" {
|
||||
respElements["content-length"] = args.RespElements["content-length"]
|
||||
}
|
||||
|
||||
@@ -1,498 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"strings"
|
||||
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/sio"
|
||||
)
|
||||
|
||||
const (
|
||||
// Represents Cache format json holding details on all other cache drives in use.
|
||||
formatCache = "cache"
|
||||
|
||||
// formatCacheV1.Cache.Version
|
||||
formatCacheVersionV1 = "1"
|
||||
formatCacheVersionV2 = "2"
|
||||
|
||||
formatMetaVersion1 = "1"
|
||||
|
||||
formatCacheV1DistributionAlgo = "CRCMOD"
|
||||
)
|
||||
|
||||
// Represents the current cache structure with list of
|
||||
// disks comprising the disk cache
|
||||
// formatCacheV1 - structure holds format config version '1'.
|
||||
type formatCacheV1 struct {
|
||||
formatMetaV1
|
||||
Cache struct {
|
||||
Version string `json:"version"` // Version of 'cache' format.
|
||||
This string `json:"this"` // This field carries assigned disk uuid.
|
||||
// Disks field carries the input disk order generated the first
|
||||
// time when fresh disks were supplied.
|
||||
Disks []string `json:"disks"`
|
||||
// Distribution algorithm represents the hashing algorithm
|
||||
// to pick the right set index for an object.
|
||||
DistributionAlgo string `json:"distributionAlgo"`
|
||||
} `json:"cache"` // Cache field holds cache format.
|
||||
}
|
||||
|
||||
// formatCacheV2 is same as formatCacheV1
|
||||
type formatCacheV2 = formatCacheV1
|
||||
|
||||
// Used to detect the version of "cache" format.
|
||||
type formatCacheVersionDetect struct {
|
||||
Cache struct {
|
||||
Version string `json:"version"`
|
||||
} `json:"cache"`
|
||||
}
|
||||
|
||||
// Return a slice of format, to be used to format uninitialized disks.
|
||||
func newFormatCacheV2(drives []string) []*formatCacheV2 {
|
||||
diskCount := len(drives)
|
||||
disks := make([]string, diskCount)
|
||||
|
||||
formats := make([]*formatCacheV2, diskCount)
|
||||
|
||||
for i := 0; i < diskCount; i++ {
|
||||
format := &formatCacheV2{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV2
|
||||
format.Cache.DistributionAlgo = formatCacheV1DistributionAlgo
|
||||
format.Cache.This = mustGetUUID()
|
||||
formats[i] = format
|
||||
disks[i] = formats[i].Cache.This
|
||||
}
|
||||
for i := 0; i < diskCount; i++ {
|
||||
format := formats[i]
|
||||
format.Cache.Disks = disks
|
||||
}
|
||||
return formats
|
||||
}
|
||||
|
||||
// Returns formatCache.Cache.Version
|
||||
func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
|
||||
format := &formatCacheVersionDetect{}
|
||||
if err := jsonLoad(r, format); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return format.Cache.Version, nil
|
||||
}
|
||||
|
||||
// Creates a new cache format.json if unformatted.
|
||||
func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
|
||||
// open file using READ & WRITE permission
|
||||
file, err := os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Close the locked file upon return.
|
||||
defer file.Close()
|
||||
|
||||
fi, err := file.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if fi.Size() != 0 {
|
||||
// format.json already got created because of another minio process's createFormatCache()
|
||||
return nil
|
||||
}
|
||||
return jsonSave(file, format)
|
||||
}
|
||||
|
||||
// This function creates a cache format file on disk and returns a slice
|
||||
// of format cache config
|
||||
func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV2, err error) {
|
||||
nformats := newFormatCacheV2(drives)
|
||||
for i, drive := range drives {
|
||||
if err = os.MkdirAll(pathJoin(drive, minioMetaBucket), 0o777); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("drive", drive)
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, err
|
||||
}
|
||||
cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
|
||||
// Fresh disk - create format.json for this cfs
|
||||
if err = createFormatCache(cacheFormatPath, nformats[i]); err != nil {
|
||||
logger.GetReqInfo(ctx).AppendTags("drive", drive)
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return nformats, nil
|
||||
}
|
||||
|
||||
func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bool, error) {
|
||||
formats := make([]*formatCacheV2, len(drives))
|
||||
var formatV2 *formatCacheV2
|
||||
migrating := false
|
||||
for i, drive := range drives {
|
||||
cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
|
||||
f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
|
||||
if err != nil {
|
||||
if osIsNotExist(err) {
|
||||
continue
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
return nil, migrating, err
|
||||
}
|
||||
defer f.Close()
|
||||
format, err := formatMetaCacheV1(f)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
formatV2 = format
|
||||
if format.Cache.Version != formatCacheVersionV2 {
|
||||
migrating = true
|
||||
}
|
||||
formats[i] = formatV2
|
||||
}
|
||||
return formats, migrating, nil
|
||||
}
|
||||
|
||||
// unmarshalls the cache format.json into formatCacheV1
|
||||
func formatMetaCacheV1(r io.ReadSeeker) (*formatCacheV1, error) {
|
||||
format := &formatCacheV1{}
|
||||
if err := jsonLoad(r, format); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return format, nil
|
||||
}
|
||||
|
||||
func checkFormatCacheValue(format *formatCacheV2, migrating bool) error {
|
||||
if format.Format != formatCache {
|
||||
return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
|
||||
}
|
||||
|
||||
// during migration one or more cache drive(s) formats can be out of sync
|
||||
if migrating {
|
||||
// Validate format version and format type.
|
||||
if format.Version != formatMetaVersion1 {
|
||||
return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
|
||||
}
|
||||
if format.Cache.Version != formatCacheVersionV2 && format.Cache.Version != formatCacheVersionV1 {
|
||||
return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
// Validate format version and format type.
|
||||
if format.Version != formatMetaVersion1 {
|
||||
return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
|
||||
}
|
||||
if format.Cache.Version != formatCacheVersionV2 {
|
||||
return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func checkFormatCacheValues(migrating bool, formats []*formatCacheV2) (int, error) {
|
||||
for i, formatCache := range formats {
|
||||
if formatCache == nil {
|
||||
continue
|
||||
}
|
||||
if err := checkFormatCacheValue(formatCache, migrating); err != nil {
|
||||
return i, err
|
||||
}
|
||||
if len(formats) != len(formatCache.Cache.Disks) {
|
||||
return i, fmt.Errorf("Expected number of cache drives %d , got %d",
|
||||
len(formatCache.Cache.Disks), len(formats))
|
||||
}
|
||||
}
|
||||
return -1, nil
|
||||
}
|
||||
|
||||
// checkCacheDisksConsistency - checks if "This" disk uuid on each disk is consistent with all "Disks" slices
|
||||
// across disks.
|
||||
func checkCacheDiskConsistency(formats []*formatCacheV2) error {
|
||||
disks := make([]string, len(formats))
|
||||
// Collect currently available disk uuids.
|
||||
for index, format := range formats {
|
||||
if format == nil {
|
||||
disks[index] = ""
|
||||
continue
|
||||
}
|
||||
disks[index] = format.Cache.This
|
||||
}
|
||||
for i, format := range formats {
|
||||
if format == nil {
|
||||
continue
|
||||
}
|
||||
j := findCacheDiskIndex(disks[i], format.Cache.Disks)
|
||||
if j == -1 {
|
||||
return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s", i, j, disks[i])
|
||||
}
|
||||
if i != j {
|
||||
return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s got %s", i, j, disks[i], format.Cache.Disks[j])
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// checkCacheDisksSliceConsistency - validate cache Disks order if they are consistent.
|
||||
func checkCacheDisksSliceConsistency(formats []*formatCacheV2) error {
|
||||
var sentinelDisks []string
|
||||
// Extract first valid Disks slice.
|
||||
for _, format := range formats {
|
||||
if format == nil {
|
||||
continue
|
||||
}
|
||||
sentinelDisks = format.Cache.Disks
|
||||
break
|
||||
}
|
||||
for _, format := range formats {
|
||||
if format == nil {
|
||||
continue
|
||||
}
|
||||
currentDisks := format.Cache.Disks
|
||||
if !reflect.DeepEqual(sentinelDisks, currentDisks) {
|
||||
return errors.New("inconsistent cache drives found")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// findCacheDiskIndex returns position of cache disk in JBOD.
|
||||
func findCacheDiskIndex(disk string, disks []string) int {
|
||||
for index, uuid := range disks {
|
||||
if uuid == disk {
|
||||
return index
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// validate whether cache drives order has changed
|
||||
func validateCacheFormats(ctx context.Context, migrating bool, formats []*formatCacheV2) error {
|
||||
count := 0
|
||||
for _, format := range formats {
|
||||
if format == nil {
|
||||
count++
|
||||
}
|
||||
}
|
||||
if count == len(formats) {
|
||||
return errors.New("Cache format files missing on all drives")
|
||||
}
|
||||
if _, err := checkFormatCacheValues(migrating, formats); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
if err := checkCacheDisksSliceConsistency(formats); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
err := checkCacheDiskConsistency(formats)
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// return true if all of the list of cache drives are
|
||||
// fresh disks
|
||||
func cacheDrivesUnformatted(drives []string) bool {
|
||||
count := 0
|
||||
for _, drive := range drives {
|
||||
cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
|
||||
if _, err := os.Stat(cacheFormatPath); osIsNotExist(err) {
|
||||
count++
|
||||
}
|
||||
}
|
||||
return count == len(drives)
|
||||
}
|
||||
|
||||
// create format.json for each cache drive if fresh disk or load format from disk
|
||||
// Then validate the format for all drives in the cache to ensure order
|
||||
// of cache drives has not changed.
|
||||
func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV2, migrating bool, err error) {
|
||||
if cacheDrivesUnformatted(drives) {
|
||||
formats, err = initFormatCache(ctx, drives)
|
||||
} else {
|
||||
formats, migrating, err = loadFormatCache(ctx, drives)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
if err = validateCacheFormats(ctx, migrating, formats); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return formats, migrating, nil
|
||||
}
|
||||
|
||||
// reads cached object on disk and writes it back after adding bitrot
|
||||
// hashsum per block as per the new disk cache format.
|
||||
func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile, destDir string, metadata map[string]string) error {
|
||||
st, err := os.Stat(oldfile)
|
||||
if err != nil {
|
||||
err = osErrToFileErr(err)
|
||||
return err
|
||||
}
|
||||
readCloser, err := readCacheFileStream(oldfile, 0, st.Size())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer readCloser.Close()
|
||||
var reader io.Reader = readCloser
|
||||
|
||||
actualSize := uint64(st.Size())
|
||||
if globalCacheKMS != nil {
|
||||
reader, err = newCacheEncryptReader(ctx, readCloser, bucket, object, metadata)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
|
||||
}
|
||||
_, _, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
|
||||
return err
|
||||
}
|
||||
|
||||
// migrate cache contents from old cacheFS format to new backend format
|
||||
// new format is flat
|
||||
//
|
||||
// sha(bucket,object)/ <== dir name
|
||||
// - part.1 <== data
|
||||
// - cache.json <== metadata
|
||||
func migrateOldCache(ctx context.Context, c *diskCache) error {
|
||||
oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets")
|
||||
cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile)
|
||||
|
||||
if _, err := os.Stat(oldCacheBucketsPath); err != nil {
|
||||
// remove .minio.sys sub directories
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
|
||||
// just migrate cache format
|
||||
return migrateCacheFormatJSON(cacheFormatPath)
|
||||
}
|
||||
|
||||
buckets, err := readDir(oldCacheBucketsPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, bucket := range buckets {
|
||||
bucket = strings.TrimSuffix(bucket, SlashSeparator)
|
||||
var objMetaPaths []string
|
||||
root := path.Join(oldCacheBucketsPath, bucket)
|
||||
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
|
||||
if strings.HasSuffix(path, cacheMetaJSONFile) {
|
||||
objMetaPaths = append(objMetaPaths, path)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, oMeta := range objMetaPaths {
|
||||
objSlice := strings.SplitN(oMeta, cacheMetaJSONFile, 2)
|
||||
object := strings.TrimPrefix(objSlice[0], path.Join(oldCacheBucketsPath, bucket))
|
||||
object = strings.TrimSuffix(object, "/")
|
||||
|
||||
destdir := getCacheSHADir(c.dir, bucket, object)
|
||||
if err := os.MkdirAll(destdir, 0o777); err != nil {
|
||||
return err
|
||||
}
|
||||
prevCachedPath := path.Join(c.dir, bucket, object)
|
||||
|
||||
// get old cached metadata
|
||||
oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
|
||||
metaPath := pathJoin(destdir, cacheMetaJSONFile)
|
||||
metaBytes, err := os.ReadFile(oldMetaPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// marshal cache metadata after adding version and stat info
|
||||
meta := &cacheMeta{}
|
||||
json := jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
if err = json.Unmarshal(metaBytes, &meta); err != nil {
|
||||
return err
|
||||
}
|
||||
// move cached object to new cache directory path
|
||||
// migrate cache data and add bit-rot protection hash sum
|
||||
// at the start of each block
|
||||
if err := migrateCacheData(ctx, c, bucket, object, prevCachedPath, destdir, meta.Meta); err != nil {
|
||||
continue
|
||||
}
|
||||
stat, err := os.Stat(prevCachedPath)
|
||||
if err != nil {
|
||||
if err == errFileNotFound {
|
||||
continue
|
||||
}
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
// old cached file can now be removed
|
||||
if err := os.Remove(prevCachedPath); err != nil {
|
||||
return err
|
||||
}
|
||||
// move cached metadata after changing cache metadata version
|
||||
meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
|
||||
meta.Version = cacheMetaVersion
|
||||
meta.Stat.Size = stat.Size()
|
||||
meta.Stat.ModTime = stat.ModTime()
|
||||
jsonData, err := json.Marshal(meta)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = os.WriteFile(metaPath, jsonData, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// delete old bucket from cache, now that all contents are cleared
|
||||
removeAll(path.Join(c.dir, bucket))
|
||||
}
|
||||
|
||||
// remove .minio.sys sub directories
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
|
||||
removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
|
||||
|
||||
return migrateCacheFormatJSON(cacheFormatPath)
|
||||
}
|
||||
|
||||
func migrateCacheFormatJSON(cacheFormatPath string) error {
|
||||
// now migrate format.json
|
||||
f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
formatV1 := formatCacheV1{}
|
||||
if err := jsonLoad(f, &formatV1); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
formatV2 := &formatCacheV2{}
|
||||
formatV2.formatMetaV1 = formatV1.formatMetaV1
|
||||
formatV2.Version = formatMetaVersion1
|
||||
formatV2.Cache = formatV1.Cache
|
||||
formatV2.Cache.Version = formatCacheVersionV2
|
||||
return jsonSave(f, formatV2)
|
||||
}
|
||||
@@ -1,321 +0,0 @@
|
||||
// Copyright (c) 2015-2021 MinIO, Inc.
|
||||
//
|
||||
// This file is part of MinIO Object Storage stack
|
||||
//
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// This program is distributed in the hope that it will be useful
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestDiskCacheFormat - tests initFormatCache, formatMetaGetFormatBackendCache, formatCacheGetVersion.
|
||||
func TestDiskCacheFormat(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
fsDirs, err := getRandomDisks(1)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
_, err = initFormatCache(ctx, fsDirs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Do the basic sanity checks to check if initFormatCache() did its job.
|
||||
cacheFormatPath := pathJoin(fsDirs[0], minioMetaBucket, formatConfigFile)
|
||||
f, err := os.OpenFile(cacheFormatPath, os.O_RDWR|os.O_SYNC, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer f.Close()
|
||||
version, err := formatCacheGetVersion(f)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if version != formatCacheVersionV2 {
|
||||
t.Fatalf(`expected: %s, got: %s`, formatCacheVersionV2, version)
|
||||
}
|
||||
|
||||
// Corrupt the format.json file and test the functions.
|
||||
// formatMetaGetFormatBackendFS, formatFSGetVersion, initFormatFS should return errors.
|
||||
if err = f.Truncate(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err = f.WriteString("b"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
|
||||
t.Fatal("expected to fail")
|
||||
}
|
||||
|
||||
// With unknown formatMetaV1.Version formatMetaGetFormatCache, initFormatCache should return error.
|
||||
if err = f.Truncate(0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Here we set formatMetaV1.Version to "2"
|
||||
if _, err = f.WriteString(`{"version":"2","format":"cache","cache":{"version":"1"}}`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
|
||||
t.Fatal("expected to fail")
|
||||
}
|
||||
}
|
||||
|
||||
// generates a valid format.json for Cache backend.
|
||||
func genFormatCacheValid() []*formatCacheV2 {
|
||||
disks := make([]string, 8)
|
||||
formatConfigs := make([]*formatCacheV2, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV1{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV2
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for Cache backend.
|
||||
func genFormatCacheInvalidVersion() []*formatCacheV2 {
|
||||
disks := make([]string, 8)
|
||||
formatConfigs := make([]*formatCacheV2, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV1{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV1
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Corrupt version numbers.
|
||||
formatConfigs[0].Version = "2"
|
||||
formatConfigs[3].Version = "-1"
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for Cache backend.
|
||||
func genFormatCacheInvalidFormat() []*formatCacheV2 {
|
||||
disks := make([]string, 8)
|
||||
formatConfigs := make([]*formatCacheV2, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV2{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV1
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Corrupt format.
|
||||
formatConfigs[0].Format = "cach"
|
||||
formatConfigs[3].Format = "cach"
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for Cache backend.
|
||||
func genFormatCacheInvalidCacheVersion() []*formatCacheV2 {
|
||||
disks := make([]string, 8)
|
||||
formatConfigs := make([]*formatCacheV2, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV2{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV1
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Corrupt version numbers.
|
||||
formatConfigs[0].Cache.Version = "10"
|
||||
formatConfigs[3].Cache.Version = "-1"
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json version for Cache backend.
|
||||
func genFormatCacheInvalidDisksCount() []*formatCacheV2 {
|
||||
disks := make([]string, 7)
|
||||
formatConfigs := make([]*formatCacheV2, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV2{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV2
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json Disks for Cache backend.
|
||||
func genFormatCacheInvalidDisks() []*formatCacheV2 {
|
||||
disks := make([]string, 8)
|
||||
formatConfigs := make([]*formatCacheV2, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV1{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV2
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
// Corrupt Disks entries on disk 6 and disk 8.
|
||||
formatConfigs[5].Cache.Disks = disks
|
||||
formatConfigs[7].Cache.Disks = disks
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json This disk UUID for Cache backend.
|
||||
func genFormatCacheInvalidThis() []*formatCacheV1 {
|
||||
disks := make([]string, 8)
|
||||
formatConfigs := make([]*formatCacheV1, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV1{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV2
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Make disk 5 and disk 8 have inconsistent disk uuid's.
|
||||
formatConfigs[4].Cache.This = mustGetUUID()
|
||||
formatConfigs[7].Cache.This = mustGetUUID()
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// generates a invalid format.json Disk UUID in wrong order for Cache backend.
|
||||
func genFormatCacheInvalidDisksOrder() []*formatCacheV2 {
|
||||
disks := make([]string, 8)
|
||||
formatConfigs := make([]*formatCacheV2, 8)
|
||||
for index := range disks {
|
||||
disks[index] = mustGetUUID()
|
||||
}
|
||||
for index := range disks {
|
||||
format := &formatCacheV1{}
|
||||
format.Version = formatMetaVersion1
|
||||
format.Format = formatCache
|
||||
format.Cache.Version = formatCacheVersionV2
|
||||
format.Cache.This = disks[index]
|
||||
format.Cache.Disks = disks
|
||||
formatConfigs[index] = format
|
||||
}
|
||||
// Re order disks for failure case.
|
||||
disks1 := make([]string, 8)
|
||||
copy(disks1, disks)
|
||||
disks1[1], disks1[2] = disks[2], disks[1]
|
||||
formatConfigs[2].Cache.Disks = disks1
|
||||
return formatConfigs
|
||||
}
|
||||
|
||||
// Wrapper for calling FormatCache tests - validates
|
||||
// - valid format
|
||||
// - unrecognized version number
|
||||
// - unrecognized format tag
|
||||
// - unrecognized cache version
|
||||
// - wrong number of Disks entries
|
||||
// - invalid This uuid
|
||||
// - invalid Disks order
|
||||
func TestFormatCache(t *testing.T) {
|
||||
formatInputCases := [][]*formatCacheV1{
|
||||
genFormatCacheValid(),
|
||||
genFormatCacheInvalidVersion(),
|
||||
genFormatCacheInvalidFormat(),
|
||||
genFormatCacheInvalidCacheVersion(),
|
||||
genFormatCacheInvalidDisksCount(),
|
||||
genFormatCacheInvalidDisks(),
|
||||
genFormatCacheInvalidThis(),
|
||||
genFormatCacheInvalidDisksOrder(),
|
||||
}
|
||||
testCases := []struct {
|
||||
formatConfigs []*formatCacheV1
|
||||
shouldPass bool
|
||||
}{
|
||||
{
|
||||
formatConfigs: formatInputCases[0],
|
||||
shouldPass: true,
|
||||
},
|
||||
{
|
||||
formatConfigs: formatInputCases[1],
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
formatConfigs: formatInputCases[2],
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
formatConfigs: formatInputCases[3],
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
formatConfigs: formatInputCases[4],
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
formatConfigs: formatInputCases[5],
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
formatConfigs: formatInputCases[6],
|
||||
shouldPass: false,
|
||||
},
|
||||
{
|
||||
formatConfigs: formatInputCases[7],
|
||||
shouldPass: false,
|
||||
},
|
||||
}
|
||||
|
||||
for i, testCase := range testCases {
|
||||
err := validateCacheFormats(context.Background(), false, testCase.formatConfigs)
|
||||
if err != nil && testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err)
|
||||
}
|
||||
if err == nil && !testCase.shouldPass {
|
||||
t.Errorf("Test %d: Expected to fail but passed instead", i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -18,20 +18,14 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/cli"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/pkg/sftp"
|
||||
ftp "goftp.io/server/v2"
|
||||
"golang.org/x/crypto/ssh"
|
||||
)
|
||||
|
||||
var globalRemoteFTPClientTransport = NewRemoteTargetHTTPTransport(true)()
|
||||
@@ -75,196 +69,6 @@ func (log *minioLogger) PrintResponse(sessionID string, code int, message string
|
||||
}
|
||||
}
|
||||
|
||||
func acceptSFTPConnection(conn net.Conn, config *ssh.ServerConfig) {
|
||||
// For SSH handshake keep a 2 minute deadline, as OpensSSH default.
|
||||
conn.SetDeadline(time.Now().Add(2 * time.Minute))
|
||||
|
||||
// Before use, a handshake must be performed on the incoming net.Conn.
|
||||
sconn, chans, reqs, err := ssh.NewServerConn(conn, config)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Once we are done with SSH handshake, remove deadline.
|
||||
conn.SetDeadline(time.Time{})
|
||||
|
||||
// The incoming Request channel must be serviced.
|
||||
go ssh.DiscardRequests(reqs)
|
||||
|
||||
// Service the incoming Channel channel.
|
||||
for newChannel := range chans {
|
||||
// Channels have a type, depending on the application level
|
||||
// protocol intended. In the case of an SFTP session, this is "subsystem"
|
||||
// with a payload string of "<length=4>sftp"
|
||||
if newChannel.ChannelType() != "session" {
|
||||
newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
|
||||
continue
|
||||
}
|
||||
|
||||
channel, requests, err := newChannel.Accept()
|
||||
if err != nil {
|
||||
logger.LogOnceIf(context.Background(), fmt.Errorf("unable to accept the connection requests channel: %w", err), "accept-channel-sftp")
|
||||
continue
|
||||
}
|
||||
|
||||
// Sessions have out-of-band requests such as "shell",
|
||||
// "pty-req" and "env". Here we handle only the
|
||||
// "subsystem" request.
|
||||
go func(in <-chan *ssh.Request) {
|
||||
for req := range in {
|
||||
ok := false
|
||||
if req.Type == "subsystem" {
|
||||
if len(req.Payload) > 4 && string(req.Payload[4:]) == "sftp" {
|
||||
ok = true
|
||||
go handleSFTPConnection(channel, sconn)
|
||||
}
|
||||
}
|
||||
|
||||
if req.WantReply {
|
||||
// We only reply to SSH packets that have `sftp` payload, all other
|
||||
// packets are rejected
|
||||
req.Reply(ok, nil)
|
||||
}
|
||||
}
|
||||
}(requests)
|
||||
}
|
||||
}
|
||||
|
||||
func handleSFTPConnection(channel ssh.Channel, sconn *ssh.ServerConn) {
|
||||
// Create the server instance for the channel using the handler we created above.
|
||||
server := sftp.NewRequestServer(channel, NewSFTPDriver(sconn.Permissions), sftp.WithRSAllocator())
|
||||
defer server.Close()
|
||||
|
||||
server.Serve()
|
||||
}
|
||||
|
||||
func startSFTPServer(c *cli.Context) {
|
||||
args := c.StringSlice("sftp")
|
||||
|
||||
var (
|
||||
port int
|
||||
publicIP string
|
||||
sshPrivateKey string
|
||||
)
|
||||
|
||||
var err error
|
||||
for _, arg := range args {
|
||||
tokens := strings.SplitN(arg, "=", 2)
|
||||
if len(tokens) != 2 {
|
||||
logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s", arg), "unable to start SFTP server")
|
||||
}
|
||||
switch tokens[0] {
|
||||
case "address":
|
||||
host, portStr, err := net.SplitHostPort(tokens[1])
|
||||
if err != nil {
|
||||
logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s (%v)", arg, err), "unable to start SFTP server")
|
||||
}
|
||||
port, err = strconv.Atoi(portStr)
|
||||
if err != nil {
|
||||
logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s (%v)", arg, err), "unable to start SFTP server")
|
||||
}
|
||||
if port < 1 || port > 65535 {
|
||||
logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s, (port number must be between 1 to 65535)", arg), "unable to start SFTP server")
|
||||
}
|
||||
publicIP = host
|
||||
case "ssh-private-key":
|
||||
sshPrivateKey = tokens[1]
|
||||
}
|
||||
}
|
||||
|
||||
if port == 0 {
|
||||
port = 8022 // Default SFTP port, since no port was given.
|
||||
}
|
||||
|
||||
if sshPrivateKey == "" {
|
||||
logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is mandatory for --sftp='ssh-private-key=path/to/id_ecdsa'"), "unable to start SFTP server")
|
||||
}
|
||||
|
||||
privateBytes, err := os.ReadFile(sshPrivateKey)
|
||||
if err != nil {
|
||||
logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is not accessible: %v", err), "unable to start SFTP server")
|
||||
}
|
||||
|
||||
private, err := ssh.ParsePrivateKey(privateBytes)
|
||||
if err != nil {
|
||||
logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is not parseable: %v", err), "unable to start SFTP server")
|
||||
}
|
||||
|
||||
// An SSH server is represented by a ServerConfig, which holds
|
||||
// certificate details and handles authentication of ServerConns.
|
||||
config := &ssh.ServerConfig{
|
||||
PasswordCallback: func(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) {
|
||||
if globalIAMSys.LDAPConfig.Enabled() {
|
||||
targetUser, targetGroups, err := globalIAMSys.LDAPConfig.Bind(c.User(), string(pass))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ldapPolicies, _ := globalIAMSys.PolicyDBGet(targetUser, false, targetGroups...)
|
||||
if len(ldapPolicies) == 0 {
|
||||
return nil, errAuthentication
|
||||
}
|
||||
return &ssh.Permissions{
|
||||
CriticalOptions: map[string]string{
|
||||
ldapUser: targetUser,
|
||||
ldapUserN: c.User(),
|
||||
},
|
||||
Extensions: make(map[string]string),
|
||||
}, nil
|
||||
}
|
||||
|
||||
ui, ok := globalIAMSys.GetUser(context.Background(), c.User())
|
||||
if !ok {
|
||||
return nil, errNoSuchUser
|
||||
}
|
||||
|
||||
if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) == 1 {
|
||||
return &ssh.Permissions{
|
||||
CriticalOptions: map[string]string{
|
||||
"accessKey": c.User(),
|
||||
},
|
||||
Extensions: make(map[string]string),
|
||||
}, nil
|
||||
}
|
||||
return nil, errAuthentication
|
||||
},
|
||||
}
|
||||
|
||||
config.AddHostKey(private)
|
||||
|
||||
// Once a ServerConfig has been configured, connections can be accepted.
|
||||
listener, err := net.Listen("tcp", net.JoinHostPort(publicIP, strconv.Itoa(port)))
|
||||
if err != nil {
|
||||
logger.Fatal(err, "unable to start listening on --sftp='port=%d'", port)
|
||||
}
|
||||
|
||||
logger.Info(fmt.Sprintf("MinIO SFTP Server listening on %s", net.JoinHostPort(publicIP, strconv.Itoa(port))))
|
||||
|
||||
var tempDelay time.Duration // how long to sleep on accept failure
|
||||
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil {
|
||||
// From https://github.com/golang/go/blob/4aa1efed4853ea067d665a952eee77c52faac774/src/net/http/server.go#L3046
|
||||
if ne, ok := err.(net.Error); ok && ne.Temporary() {
|
||||
if tempDelay == 0 {
|
||||
tempDelay = 5 * time.Millisecond
|
||||
} else {
|
||||
tempDelay *= 2
|
||||
}
|
||||
if max := 1 * time.Second; tempDelay > max {
|
||||
tempDelay = max
|
||||
}
|
||||
logger.LogOnceIf(context.Background(), fmt.Errorf("error while accepting connections: %w, retrying in %s", err, tempDelay), "accept-limit-sftp")
|
||||
time.Sleep(tempDelay)
|
||||
continue
|
||||
}
|
||||
logger.Fatal(err, "unrecoverable error while accepting new connections")
|
||||
}
|
||||
|
||||
go acceptSFTPConnection(conn, config)
|
||||
}
|
||||
}
|
||||
|
||||
func startFTPServer(c *cli.Context) {
|
||||
args := c.StringSlice("ftp")
|
||||
|
||||
|
||||
+4
-6
@@ -309,7 +309,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
Remove: healDeleteDangling,
|
||||
})
|
||||
if err != nil {
|
||||
if isErrObjectNotFound(err) {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
// queueing happens across namespace, ignore
|
||||
// objects that are not found.
|
||||
return
|
||||
@@ -335,7 +335,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
ScanMode: scanMode,
|
||||
Remove: healDeleteDangling,
|
||||
}); err != nil {
|
||||
if isErrObjectNotFound(err) {
|
||||
if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
|
||||
// queueing happens across namespace, ignore
|
||||
// objects that are not found.
|
||||
versionNotFound++
|
||||
@@ -344,9 +344,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
// If not deleted, assume they failed.
|
||||
result = healEntryFailure(uint64(version.Size))
|
||||
if version.VersionID != "" {
|
||||
if !isErrVersionNotFound(err) {
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
|
||||
} else {
|
||||
logger.LogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, version.Name, err))
|
||||
}
|
||||
@@ -358,7 +356,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
|
||||
return
|
||||
}
|
||||
}
|
||||
// All versions resulted in 'ObjectNotFound'
|
||||
// All versions resulted in 'ObjectNotFound/VersionNotFound'
|
||||
if versionNotFound == len(fivs.Versions) {
|
||||
return
|
||||
}
|
||||
|
||||
+9
-9
@@ -23,7 +23,6 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/console/restapi"
|
||||
@@ -35,10 +34,10 @@ import (
|
||||
"github.com/minio/minio/internal/config"
|
||||
"github.com/minio/minio/internal/handlers"
|
||||
"github.com/minio/minio/internal/kms"
|
||||
"go.uber.org/atomic"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/internal/auth"
|
||||
"github.com/minio/minio/internal/config/cache"
|
||||
"github.com/minio/minio/internal/config/callhome"
|
||||
"github.com/minio/minio/internal/config/compress"
|
||||
"github.com/minio/minio/internal/config/dns"
|
||||
@@ -273,12 +272,6 @@ var (
|
||||
globalBucketQuotaSys *BucketQuotaSys
|
||||
globalBucketVersioningSys *BucketVersioningSys
|
||||
|
||||
// Disk cache drives
|
||||
globalCacheConfig cache.Config
|
||||
|
||||
// Initialized KMS configuration for disk cache
|
||||
globalCacheKMS kms.KMS
|
||||
|
||||
// Allocated etcd endpoint for config and bucket DNS.
|
||||
globalEtcdClient *etcd.Client
|
||||
|
||||
@@ -320,7 +313,14 @@ var (
|
||||
globalAuthZPlugin *polplugin.AuthZPlugin
|
||||
|
||||
// Deployment ID - unique per deployment
|
||||
globalDeploymentID string
|
||||
globalDeploymentIDPtr atomic.Pointer[string]
|
||||
globalDeploymentID = func() string {
|
||||
ptr := globalDeploymentIDPtr.Load()
|
||||
if ptr == nil {
|
||||
return ""
|
||||
}
|
||||
return *ptr
|
||||
}
|
||||
|
||||
globalAllHealState *allHealState
|
||||
|
||||
|
||||
+33
-15
@@ -90,21 +90,15 @@ func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
}
|
||||
|
||||
// ReadinessCheckHandler Checks if the process is up. Always returns success.
|
||||
// ReadinessCheckHandler checks whether MinIO is up and ready to serve requests.
|
||||
// It also checks whether the KMS is available and whether etcd is reachable,
|
||||
// if configured.
|
||||
func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
LivenessCheckHandler(w, r)
|
||||
}
|
||||
|
||||
// LivenessCheckHandler - Checks if the process is up. Always returns success.
|
||||
func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
objLayer := newObjectLayerFn()
|
||||
if objLayer == nil {
|
||||
// Service not initialized yet
|
||||
w.Header().Set(xhttp.MinIOServerStatus, unavailable)
|
||||
if objLayer := newObjectLayerFn(); objLayer == nil {
|
||||
w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet
|
||||
}
|
||||
|
||||
peerCall := r.Header.Get(xhttp.MinIOPeerCall) != ""
|
||||
if peerCall {
|
||||
if r.Header.Get(xhttp.MinIOPeerCall) != "" {
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
return
|
||||
}
|
||||
|
||||
@@ -120,7 +114,7 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
|
||||
// Verify if KMS is reachable if its configured
|
||||
if GlobalKMS != nil && !peerCall {
|
||||
if GlobalKMS != nil {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
|
||||
defer cancel()
|
||||
|
||||
@@ -153,6 +147,30 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
}
|
||||
|
||||
// LivenessCheckHandler checks whether MinIO is up. It differs from the
|
||||
// readiness handler since a failing liveness check causes pod restarts
|
||||
// in K8S enviromnents. Therefore, it does not contact external systems.
|
||||
func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
|
||||
if objLayer := newObjectLayerFn(); objLayer == nil {
|
||||
w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet
|
||||
}
|
||||
if r.Header.Get(xhttp.MinIOPeerCall) != "" {
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
return
|
||||
}
|
||||
|
||||
if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
|
||||
apiErr := getAPIError(ErrBusy)
|
||||
switch r.Method {
|
||||
case http.MethodHead:
|
||||
writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
|
||||
case http.MethodGet:
|
||||
writeErrorResponse(r.Context(), w, apiErr, r.URL)
|
||||
}
|
||||
return
|
||||
}
|
||||
writeResponse(w, http.StatusOK, nil, mimeNone)
|
||||
}
|
||||
|
||||
+7
-17
@@ -112,14 +112,9 @@ func saveIAMFormat(ctx context.Context, store IAMStorageAPI) error {
|
||||
bootstrapTraceMsg("Load IAM format file")
|
||||
var iamFmt iamFormat
|
||||
path := getIAMFormatFilePath()
|
||||
if err := store.loadIAMConfig(ctx, &iamFmt, path); err != nil {
|
||||
switch err {
|
||||
case errConfigNotFound:
|
||||
// Need to migrate to V1.
|
||||
default:
|
||||
// if IAM format
|
||||
return err
|
||||
}
|
||||
if err := store.loadIAMConfig(ctx, &iamFmt, path); err != nil && !errors.Is(err, errConfigNotFound) {
|
||||
// if IAM format
|
||||
return err
|
||||
}
|
||||
|
||||
if iamFmt.Version >= iamFormatVersion1 {
|
||||
@@ -129,12 +124,7 @@ func saveIAMFormat(ctx context.Context, store IAMStorageAPI) error {
|
||||
|
||||
bootstrapTraceMsg("Write IAM format file")
|
||||
// Save iam format to version 1.
|
||||
if err := store.saveIAMConfig(ctx, newIAMFormatVersion1(), path); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
return store.saveIAMConfig(ctx, newIAMFormatVersion1(), path)
|
||||
}
|
||||
|
||||
func getGroupInfoPath(group string) string {
|
||||
@@ -430,10 +420,10 @@ func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
|
||||
}
|
||||
u.Credentials.Claims = jwtClaims.Map()
|
||||
}
|
||||
if !u.Credentials.IsTemp() {
|
||||
c.iamUsersMap[key] = u
|
||||
} else {
|
||||
if u.Credentials.IsTemp() && !u.Credentials.IsServiceAccount() {
|
||||
c.iamSTSAccountsMap[key] = u
|
||||
} else {
|
||||
c.iamUsersMap[key] = u
|
||||
}
|
||||
c.updatedAt = time.Now()
|
||||
return nil
|
||||
|
||||
+18
-10
@@ -293,6 +293,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
|
||||
if err := saveIAMFormat(retryCtx, sys.store); err != nil {
|
||||
if configRetriableErrors(err) {
|
||||
logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
|
||||
time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
|
||||
continue
|
||||
}
|
||||
logger.LogIf(ctx, fmt.Errorf("IAM sub-system is partially initialized, unable to write the IAM format: %w", err))
|
||||
@@ -307,7 +308,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
|
||||
if err := sys.Load(retryCtx, true); err != nil {
|
||||
if configRetriableErrors(err) {
|
||||
logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
|
||||
time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
|
||||
time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
|
||||
continue
|
||||
}
|
||||
if err != nil {
|
||||
@@ -1662,18 +1663,25 @@ func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool,
|
||||
return
|
||||
}
|
||||
if dn == "" {
|
||||
err = errNoSuchUser
|
||||
return
|
||||
// Still attempt to detach if provided user is a DN.
|
||||
if !isAttach && sys.LDAPConfig.IsLDAPUserDN(r.User) {
|
||||
dn = r.User
|
||||
} else {
|
||||
err = errNoSuchUser
|
||||
return
|
||||
}
|
||||
}
|
||||
isGroup = false
|
||||
} else {
|
||||
var exists bool
|
||||
if exists, err = sys.LDAPConfig.DoesGroupDNExist(r.Group); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
} else if !exists {
|
||||
err = errNoSuchGroup
|
||||
return
|
||||
if isAttach {
|
||||
var exists bool
|
||||
if exists, err = sys.LDAPConfig.DoesGroupDNExist(r.Group); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return
|
||||
} else if !exists {
|
||||
err = errNoSuchGroup
|
||||
return
|
||||
}
|
||||
}
|
||||
dn = r.Group
|
||||
isGroup = true
|
||||
|
||||
@@ -251,10 +251,8 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
||||
meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
|
||||
meta.name = pathJoinBuf(sb, current, meta.name)
|
||||
meta.name = decodeDirObject(meta.name)
|
||||
if err := send(meta); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
return send(meta)
|
||||
}
|
||||
// Check legacy.
|
||||
if HasSuffix(entry, xlStorageFormatFileV1) && legacy {
|
||||
@@ -270,10 +268,8 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
|
||||
meta.name = strings.TrimSuffix(entry, xlStorageFormatFileV1)
|
||||
meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
|
||||
meta.name = pathJoinBuf(sb, current, meta.name)
|
||||
if err := send(meta); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
|
||||
return send(meta)
|
||||
}
|
||||
// Skip all other files.
|
||||
}
|
||||
|
||||
+6
-19
@@ -20,7 +20,6 @@ package cmd
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/minio/madmin-go/v3"
|
||||
@@ -42,18 +41,6 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
|
||||
return
|
||||
}
|
||||
|
||||
byHostName := globalMinioAddr
|
||||
if len(opts.hosts) > 0 {
|
||||
server := getLocalServerProperty(globalEndpoints, &http.Request{
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
if _, ok := opts.hosts[server.Endpoint]; ok {
|
||||
byHostName = server.Endpoint
|
||||
} else {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if types.Contains(madmin.MetricsDisk) {
|
||||
m.ByDisk = make(map[string]madmin.DiskMetric)
|
||||
aggr := madmin.DiskMetric{
|
||||
@@ -87,7 +74,7 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
|
||||
}
|
||||
netStats, err := net.GetInterfaceNetStats(globalInternodeInterface)
|
||||
if err != nil {
|
||||
m.Errors = append(m.Errors, err.Error())
|
||||
m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (nicstats)", globalMinioAddr, err.Error()))
|
||||
} else {
|
||||
m.Aggregated.Net.NetStats = netStats
|
||||
}
|
||||
@@ -104,18 +91,18 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
|
||||
}
|
||||
cm, err := c.Times(false)
|
||||
if err != nil {
|
||||
m.Errors = append(m.Errors, err.Error())
|
||||
m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cputimes)", globalMinioAddr, err.Error()))
|
||||
} else {
|
||||
// not collecting per-cpu stats, so there will be only one element
|
||||
if len(cm) == 1 {
|
||||
m.Aggregated.CPU.TimesStat = &cm[0]
|
||||
} else {
|
||||
m.Errors = append(m.Errors, fmt.Sprintf("Expected one CPU stat, got %d", len(cm)))
|
||||
m.Errors = append(m.Errors, fmt.Sprintf("%s: Expected one CPU stat, got %d", globalMinioAddr, len(cm)))
|
||||
}
|
||||
}
|
||||
loadStat, err := load.Avg()
|
||||
if err != nil {
|
||||
m.Errors = append(m.Errors, err.Error())
|
||||
m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (loadStat)", globalMinioAddr, err.Error()))
|
||||
} else {
|
||||
m.Aggregated.CPU.LoadStat = loadStat
|
||||
}
|
||||
@@ -123,8 +110,8 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
|
||||
// Add types...
|
||||
|
||||
// ByHost is a shallow reference, so careful about sharing.
|
||||
m.ByHost = map[string]madmin.Metrics{byHostName: m.Aggregated}
|
||||
m.Hosts = append(m.Hosts, byHostName)
|
||||
m.ByHost = map[string]madmin.Metrics{globalMinioAddr: m.Aggregated}
|
||||
m.Hosts = append(m.Hosts, globalMinioAddr)
|
||||
|
||||
return m
|
||||
}
|
||||
|
||||
+12
-6
@@ -74,7 +74,8 @@ const (
|
||||
var (
|
||||
resourceCollector *minioResourceCollector
|
||||
// resourceMetricsMap is a map of subsystem to its metrics
|
||||
resourceMetricsMap map[MetricSubsystem]ResourceMetrics
|
||||
resourceMetricsMap map[MetricSubsystem]ResourceMetrics
|
||||
resourceMetricsMapMu sync.RWMutex
|
||||
// resourceMetricsHelpMap maps metric name to its help string
|
||||
resourceMetricsHelpMap map[MetricName]string
|
||||
resourceMetricsGroups []*MetricsGroup
|
||||
@@ -159,6 +160,8 @@ func init() {
|
||||
}
|
||||
|
||||
func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64, labels map[string]string, isCumulative bool) {
|
||||
resourceMetricsMapMu.Lock()
|
||||
defer resourceMetricsMapMu.Unlock()
|
||||
subsysMetrics, found := resourceMetricsMap[subSys]
|
||||
if !found {
|
||||
subsysMetrics = ResourceMetrics{}
|
||||
@@ -320,7 +323,9 @@ func collectLocalResourceMetrics() {
|
||||
|
||||
// startResourceMetricsCollection - starts the job for collecting resource metrics
|
||||
func startResourceMetricsCollection() {
|
||||
resourceMetricsMapMu.Lock()
|
||||
resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{}
|
||||
resourceMetricsMapMu.Unlock()
|
||||
metricsTimer := time.NewTimer(resourceMetricsCollectionInterval)
|
||||
defer metricsTimer.Stop()
|
||||
|
||||
@@ -382,12 +387,11 @@ func newMinioResourceCollector(metricsGroups []*MetricsGroup) *minioResourceColl
|
||||
func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAvgMax bool) []Metric {
|
||||
help := resourceMetricsHelpMap[rm.Name]
|
||||
name := rm.Name
|
||||
|
||||
metrics := []Metric{}
|
||||
metrics := make([]Metric, 0, 3)
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getResourceMetricDescription(subSys, name, help),
|
||||
Value: rm.Current,
|
||||
VariableLabels: rm.Labels,
|
||||
VariableLabels: cloneMSS(rm.Labels),
|
||||
})
|
||||
|
||||
if requireAvgMax {
|
||||
@@ -396,7 +400,7 @@ func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAv
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getResourceMetricDescription(subSys, avgName, avgHelp),
|
||||
Value: math.Round(rm.Avg*100) / 100,
|
||||
VariableLabels: rm.Labels,
|
||||
VariableLabels: cloneMSS(rm.Labels),
|
||||
})
|
||||
|
||||
maxName := MetricName(fmt.Sprintf("%s_max", name))
|
||||
@@ -404,7 +408,7 @@ func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAv
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getResourceMetricDescription(subSys, maxName, maxHelp),
|
||||
Value: rm.Max,
|
||||
VariableLabels: rm.Labels,
|
||||
VariableLabels: cloneMSS(rm.Labels),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -429,6 +433,8 @@ func getResourceMetrics() *MetricsGroup {
|
||||
metrics := []Metric{}
|
||||
|
||||
subSystems := []MetricSubsystem{interfaceSubsystem, memSubsystem, driveSubsystem, cpuSubsystem}
|
||||
resourceMetricsMapMu.RLock()
|
||||
defer resourceMetricsMapMu.RUnlock()
|
||||
for _, subSys := range subSystems {
|
||||
stats, found := resourceMetricsMap[subSys]
|
||||
if found {
|
||||
|
||||
+64
-142
@@ -22,6 +22,7 @@ import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -59,7 +60,6 @@ func init() {
|
||||
}
|
||||
|
||||
peerMetricsGroups = []*MetricsGroup{
|
||||
getCacheMetrics(),
|
||||
getGoMetrics(),
|
||||
getHTTPMetrics(false),
|
||||
getNotificationMetrics(),
|
||||
@@ -85,7 +85,6 @@ func init() {
|
||||
|
||||
nodeGroups := []*MetricsGroup{
|
||||
getNodeHealthMetrics(),
|
||||
getCacheMetrics(),
|
||||
getHTTPMetrics(false),
|
||||
getNetworkMetrics(),
|
||||
getMinioVersionMetrics(),
|
||||
@@ -238,8 +237,6 @@ const (
|
||||
latencyMicroSec MetricName = "latency_us"
|
||||
latencyNanoSec MetricName = "latency_ns"
|
||||
|
||||
usagePercent MetricName = "update_percent"
|
||||
|
||||
commitInfo MetricName = "commit_info"
|
||||
usageInfo MetricName = "usage_info"
|
||||
versionInfo MetricName = "version_info"
|
||||
@@ -697,7 +694,7 @@ func getClusterObjectVersionsMD() MetricDescription {
|
||||
Namespace: clusterMetricNamespace,
|
||||
Subsystem: objectsSubsystem,
|
||||
Name: versionDistribution,
|
||||
Help: "Distribution of object sizes across a cluster",
|
||||
Help: "Distribution of object versions across a cluster",
|
||||
Type: histogramMetric,
|
||||
}
|
||||
}
|
||||
@@ -1230,76 +1227,6 @@ func getS3RejectedInvalidRequestsTotalMD() MetricDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func getCacheHitsTotalMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: cacheSubsystem,
|
||||
Name: hitsTotal,
|
||||
Help: "Total number of drive cache hits",
|
||||
Type: counterMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getCacheHitsMissedTotalMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: cacheSubsystem,
|
||||
Name: missedTotal,
|
||||
Help: "Total number of drive cache misses",
|
||||
Type: counterMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getCacheUsagePercentMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: minioNamespace,
|
||||
Name: usagePercent,
|
||||
Help: "Total percentage cache usage",
|
||||
Type: gaugeMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getCacheUsageInfoMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: cacheSubsystem,
|
||||
Name: usageInfo,
|
||||
Help: "Total percentage cache usage, value of 1 indicates high and 0 low, label level is set as well",
|
||||
Type: gaugeMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getCacheUsedBytesMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: cacheSubsystem,
|
||||
Name: usedBytes,
|
||||
Help: "Current cache usage in bytes",
|
||||
Type: gaugeMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getCacheTotalBytesMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: cacheSubsystem,
|
||||
Name: totalBytes,
|
||||
Help: "Total size of cache drive in bytes",
|
||||
Type: gaugeMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getCacheSentBytesMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: cacheSubsystem,
|
||||
Name: sentBytes,
|
||||
Help: "Total number of bytes served from cache",
|
||||
Type: counterMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getHealObjectsTotalMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: healMetricNamespace,
|
||||
@@ -2454,56 +2381,6 @@ func getObjectsScanned(seq *healSequence) (m []Metric) {
|
||||
return
|
||||
}
|
||||
|
||||
func getCacheMetrics() *MetricsGroup {
|
||||
mg := &MetricsGroup{
|
||||
cacheInterval: 10 * time.Second,
|
||||
}
|
||||
mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
|
||||
cacheObjLayer := newCachedObjectLayerFn()
|
||||
// Service not initialized yet
|
||||
if cacheObjLayer == nil {
|
||||
return
|
||||
}
|
||||
metrics = make([]Metric, 0, 20)
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getCacheHitsTotalMD(),
|
||||
Value: float64(cacheObjLayer.CacheStats().getHits()),
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getCacheHitsMissedTotalMD(),
|
||||
Value: float64(cacheObjLayer.CacheStats().getMisses()),
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getCacheSentBytesMD(),
|
||||
Value: float64(cacheObjLayer.CacheStats().getBytesServed()),
|
||||
})
|
||||
for _, cdStats := range cacheObjLayer.CacheStats().GetDiskStats() {
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getCacheUsagePercentMD(),
|
||||
Value: float64(cdStats.UsagePercent),
|
||||
VariableLabels: map[string]string{"drive": cdStats.Dir},
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getCacheUsageInfoMD(),
|
||||
Value: float64(cdStats.UsageState),
|
||||
VariableLabels: map[string]string{"drive": cdStats.Dir, "level": cdStats.GetUsageLevelString()},
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getCacheUsedBytesMD(),
|
||||
Value: float64(cdStats.UsageSize),
|
||||
VariableLabels: map[string]string{"drive": cdStats.Dir},
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getCacheTotalBytesMD(),
|
||||
Value: float64(cdStats.TotalCapacity),
|
||||
VariableLabels: map[string]string{"drive": cdStats.Dir},
|
||||
})
|
||||
}
|
||||
return
|
||||
})
|
||||
return mg
|
||||
}
|
||||
|
||||
func getDistLockMetrics() *MetricsGroup {
|
||||
mg := &MetricsGroup{
|
||||
cacheInterval: 1 * time.Second,
|
||||
@@ -2556,30 +2433,53 @@ func getNotificationMetrics() *MetricsGroup {
|
||||
cacheInterval: 10 * time.Second,
|
||||
}
|
||||
mg.RegisterRead(func(ctx context.Context) []Metric {
|
||||
nstats := globalNotifyTargetList.Stats()
|
||||
metrics := make([]Metric, 0, 1+len(nstats.TargetStats))
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: notifySubsystem,
|
||||
Name: "current_send_in_progress",
|
||||
Help: "Number of concurrent async Send calls active to all targets",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
Value: float64(nstats.CurrentSendCalls),
|
||||
})
|
||||
for _, st := range nstats.TargetStats {
|
||||
metrics := make([]Metric, 0, 3)
|
||||
|
||||
if globalEventNotifier != nil {
|
||||
nstats := globalEventNotifier.targetList.Stats()
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: notifySubsystem,
|
||||
Name: "target_queue_length",
|
||||
Help: "Number of unsent notifications in queue for target",
|
||||
Name: "current_send_in_progress",
|
||||
Help: "Number of concurrent async Send calls active to all targets",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
|
||||
Value: float64(st.CurrentQueue),
|
||||
Value: float64(nstats.CurrentSendCalls),
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: notifySubsystem,
|
||||
Name: "events_skipped_total",
|
||||
Help: "Events that were skipped due to full queue",
|
||||
Type: counterMetric,
|
||||
},
|
||||
Value: float64(nstats.EventsSkipped),
|
||||
})
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: notifySubsystem,
|
||||
Name: "events_sent_total",
|
||||
Help: "Total number of events sent since start",
|
||||
Type: counterMetric,
|
||||
},
|
||||
Value: float64(nstats.TotalEvents),
|
||||
})
|
||||
for _, st := range nstats.TargetStats {
|
||||
metrics = append(metrics, Metric{
|
||||
Description: MetricDescription{
|
||||
Namespace: minioNamespace,
|
||||
Subsystem: notifySubsystem,
|
||||
Name: "target_queue_length",
|
||||
Help: "Number of unsent notifications in queue for target",
|
||||
Type: gaugeMetric,
|
||||
},
|
||||
VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name},
|
||||
Value: float64(st.CurrentQueue),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
lstats := globalLambdaTargetList.Stats()
|
||||
@@ -3288,6 +3188,16 @@ func getClusterHealthStatusMD() MetricDescription {
|
||||
}
|
||||
}
|
||||
|
||||
func getClusterErasureSetToleranceMD() MetricDescription {
|
||||
return MetricDescription{
|
||||
Namespace: clusterMetricNamespace,
|
||||
Subsystem: "health",
|
||||
Name: "erasure_set_tolerance",
|
||||
Help: "Get erasure set tolerance status",
|
||||
Type: gaugeMetric,
|
||||
}
|
||||
}
|
||||
|
||||
func getClusterHealthMetrics() *MetricsGroup {
|
||||
mg := &MetricsGroup{
|
||||
cacheInterval: 10 * time.Second,
|
||||
@@ -3319,6 +3229,18 @@ func getClusterHealthMetrics() *MetricsGroup {
|
||||
Value: float64(health),
|
||||
})
|
||||
|
||||
for _, h := range result.ESHealth {
|
||||
labels := map[string]string{
|
||||
"pool": strconv.Itoa(h.PoolID),
|
||||
"set": strconv.Itoa(h.SetID),
|
||||
}
|
||||
metrics = append(metrics, Metric{
|
||||
Description: getClusterErasureSetToleranceMD(),
|
||||
VariableLabels: labels,
|
||||
Value: float64(h.HealthyDrives - h.WriteQuorum),
|
||||
})
|
||||
}
|
||||
|
||||
return
|
||||
})
|
||||
|
||||
|
||||
@@ -108,7 +108,6 @@ func (c *minioCollector) Collect(ch chan<- prometheus.Metric) {
|
||||
bucketUsageMetricsPrometheus(ch)
|
||||
networkMetricsPrometheus(ch)
|
||||
httpMetricsPrometheus(ch)
|
||||
cacheMetricsPrometheus(ch)
|
||||
healingMetricsPrometheus(ch)
|
||||
}
|
||||
|
||||
@@ -188,82 +187,6 @@ func healingMetricsPrometheus(ch chan<- prometheus.Metric) {
|
||||
}
|
||||
}
|
||||
|
||||
// collects cache metrics for MinIO server in Prometheus specific format
|
||||
// and sends to given channel
|
||||
func cacheMetricsPrometheus(ch chan<- prometheus.Metric) {
|
||||
cacheObjLayer := newCachedObjectLayerFn()
|
||||
// Service not initialized yet
|
||||
if cacheObjLayer == nil {
|
||||
return
|
||||
}
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(cacheNamespace, "hits", "total"),
|
||||
"Total number of drive cache hits in current MinIO instance",
|
||||
nil, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(cacheObjLayer.CacheStats().getHits()),
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(cacheNamespace, "misses", "total"),
|
||||
"Total number of drive cache misses in current MinIO instance",
|
||||
nil, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(cacheObjLayer.CacheStats().getMisses()),
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(cacheNamespace, "data", "served"),
|
||||
"Total number of bytes served from cache of current MinIO instance",
|
||||
nil, nil),
|
||||
prometheus.CounterValue,
|
||||
float64(cacheObjLayer.CacheStats().getBytesServed()),
|
||||
)
|
||||
for _, cdStats := range cacheObjLayer.CacheStats().GetDiskStats() {
|
||||
// Cache disk usage percentage
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(cacheNamespace, "usage", "percent"),
|
||||
"Total percentage cache usage",
|
||||
[]string{"disk"}, nil),
|
||||
prometheus.GaugeValue,
|
||||
float64(cdStats.UsagePercent),
|
||||
cdStats.Dir,
|
||||
)
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName(cacheNamespace, "usage", "high"),
|
||||
"Indicates cache usage is high or low, relative to current cache 'quota' settings",
|
||||
[]string{"disk"}, nil),
|
||||
prometheus.GaugeValue,
|
||||
float64(cdStats.UsageState),
|
||||
cdStats.Dir,
|
||||
)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("cache", "usage", "size"),
|
||||
"Indicates current cache usage in bytes",
|
||||
[]string{"disk"}, nil),
|
||||
prometheus.GaugeValue,
|
||||
float64(cdStats.UsageSize),
|
||||
cdStats.Dir,
|
||||
)
|
||||
|
||||
ch <- prometheus.MustNewConstMetric(
|
||||
prometheus.NewDesc(
|
||||
prometheus.BuildFQName("cache", "total", "size"),
|
||||
"Indicates total size of cache drive",
|
||||
[]string{"disk"}, nil),
|
||||
prometheus.GaugeValue,
|
||||
float64(cdStats.TotalCapacity),
|
||||
cdStats.Dir,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
// collects http metrics for MinIO server in Prometheus specific format
|
||||
// and sends to given channel
|
||||
func httpMetricsPrometheus(ch chan<- prometheus.Metric) {
|
||||
|
||||
+26
-1
@@ -701,6 +701,31 @@ func (sys *NotificationSys) GetCPUs(ctx context.Context) []madmin.CPUs {
|
||||
return reply
|
||||
}
|
||||
|
||||
// GetNetInfo - Network information
|
||||
func (sys *NotificationSys) GetNetInfo(ctx context.Context) []madmin.NetInfo {
|
||||
reply := make([]madmin.NetInfo, len(sys.peerClients))
|
||||
|
||||
g := errgroup.WithNErrs(len(sys.peerClients))
|
||||
for index, client := range sys.peerClients {
|
||||
if client == nil {
|
||||
continue
|
||||
}
|
||||
index := index
|
||||
g.Go(func() error {
|
||||
var err error
|
||||
reply[index], err = sys.peerClients[index].GetNetInfo(ctx)
|
||||
return err
|
||||
}, index)
|
||||
}
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
sys.addNodeErr(&reply[index], sys.peerClients[index], err)
|
||||
}
|
||||
}
|
||||
return reply
|
||||
}
|
||||
|
||||
// GetPartitions - Disk partition information
|
||||
func (sys *NotificationSys) GetPartitions(ctx context.Context) []madmin.Partitions {
|
||||
reply := make([]madmin.Partitions, len(sys.peerClients))
|
||||
@@ -777,7 +802,7 @@ func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType,
|
||||
|
||||
for index, err := range g.Wait() {
|
||||
if err != nil {
|
||||
reply[index].Errors = []string{err.Error()}
|
||||
reply[index].Errors = []string{fmt.Sprintf("%s: %s (rpc)", sys.peerClients[index].String(), err.Error())}
|
||||
}
|
||||
}
|
||||
return reply
|
||||
|
||||
@@ -52,9 +52,6 @@ var globalObjLayerMutex sync.RWMutex
|
||||
// Global object layer, only accessed by globalObjectAPI.
|
||||
var globalObjectAPI ObjectLayer
|
||||
|
||||
// Global cacheObjects, only accessed by newCacheObjectsFn().
|
||||
var globalCacheObjectAPI CacheObjectLayer
|
||||
|
||||
type storageOpts struct {
|
||||
cleanUp bool
|
||||
healthCheck bool
|
||||
|
||||
@@ -149,11 +149,6 @@ type ObjectInfo struct {
|
||||
// Date and time at which the object is no longer able to be cached
|
||||
Expires time.Time
|
||||
|
||||
// CacheStatus sets status of whether this is a cache hit/miss
|
||||
CacheStatus CacheStatusType
|
||||
// CacheLookupStatus sets whether a cacheable response is present in the cache
|
||||
CacheLookupStatus CacheStatusType
|
||||
|
||||
// Specify object storage class
|
||||
StorageClass string
|
||||
|
||||
@@ -245,8 +240,6 @@ func (o *ObjectInfo) Clone() (cinfo ObjectInfo) {
|
||||
ContentType: o.ContentType,
|
||||
ContentEncoding: o.ContentEncoding,
|
||||
Expires: o.Expires,
|
||||
CacheStatus: o.CacheStatus,
|
||||
CacheLookupStatus: o.CacheLookupStatus,
|
||||
StorageClass: o.StorageClass,
|
||||
ReplicationStatus: o.ReplicationStatus,
|
||||
UserTags: o.UserTags,
|
||||
|
||||
@@ -149,7 +149,6 @@ func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer
|
||||
// This is done on purpose since the order of errors is
|
||||
// important here bucket does not exist error should
|
||||
// happen before we return an error for invalid object name.
|
||||
// FIXME: should be moved to handler layer.
|
||||
if err := checkBucketExist(ctx, bucket, obj); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -170,16 +169,7 @@ func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer
|
||||
}
|
||||
|
||||
// Checks for PutObject arguments validity, also validates if bucket exists.
|
||||
func checkPutObjectArgs(ctx context.Context, bucket, object string, obj getBucketInfoI) error {
|
||||
// Verify if bucket exists before validating object name.
|
||||
// This is done on purpose since the order of errors is
|
||||
// important here bucket does not exist error should
|
||||
// happen before we return an error for invalid object name.
|
||||
// FIXME: should be moved to handler layer.
|
||||
if err := checkBucketExist(ctx, bucket, obj); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
func checkPutObjectArgs(ctx context.Context, bucket, object string) error {
|
||||
if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"runtime"
|
||||
"strings"
|
||||
@@ -1223,8 +1224,18 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
|
||||
t.Fatalf("%s : %s", instanceType, err.Error())
|
||||
}
|
||||
|
||||
// Remove some random disk.
|
||||
removeDiskN(disks, 1)
|
||||
z := obj.(*erasureServerPools)
|
||||
er := z.serverPools[0].sets[0]
|
||||
|
||||
erasureDisks := er.getDisks()
|
||||
ridx := rand.Intn(len(erasureDisks))
|
||||
|
||||
z.serverPools[0].erasureDisksMu.Lock()
|
||||
er.getDisks = func() []StorageAPI {
|
||||
erasureDisks[ridx] = newNaughtyDisk(erasureDisks[ridx], nil, errFaultyDisk)
|
||||
return erasureDisks
|
||||
}
|
||||
z.serverPools[0].erasureDisksMu.Unlock()
|
||||
|
||||
uploadIDs = append(uploadIDs, res.UploadID)
|
||||
|
||||
@@ -1257,9 +1268,6 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
|
||||
}
|
||||
}
|
||||
|
||||
// Remove one disk.
|
||||
removeDiskN(disks, 1)
|
||||
|
||||
partInfos := []ListPartsInfo{
|
||||
// partinfos - 0.
|
||||
{
|
||||
|
||||
+39
-55
@@ -30,7 +30,6 @@ import (
|
||||
"github.com/minio/minio/internal/crypto"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) {
|
||||
@@ -115,44 +114,43 @@ func getOpts(ctx context.Context, r *http.Request, bucket, object string) (Objec
|
||||
}
|
||||
opts.PartNumber = partNumber
|
||||
opts.VersionID = vid
|
||||
delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker))
|
||||
if delMarker != "" {
|
||||
switch delMarker {
|
||||
case "true":
|
||||
opts.DeleteMarker = true
|
||||
case "false":
|
||||
default:
|
||||
err = fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceDeleteMarker, fmt.Errorf("DeleteMarker should be true or false"))
|
||||
logger.LogIf(ctx, err)
|
||||
return opts, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
|
||||
delMarker, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOSourceDeleteMarker)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
replReadyCheck := strings.TrimSpace(r.Header.Get(xhttp.MinIOCheckDMReplicationReady))
|
||||
if replReadyCheck != "" {
|
||||
switch replReadyCheck {
|
||||
case "true":
|
||||
opts.CheckDMReplicationReady = true
|
||||
case "false":
|
||||
default:
|
||||
err = fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOCheckDMReplicationReady, fmt.Errorf("should be true or false"))
|
||||
logger.LogIf(ctx, err)
|
||||
return opts, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
opts.DeleteMarker = delMarker
|
||||
|
||||
replReadyCheck, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOCheckDMReplicationReady)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
opts.CheckDMReplicationReady = replReadyCheck
|
||||
|
||||
opts.Tagging = r.Header.Get(xhttp.AmzTagDirective) == accessDirective
|
||||
opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object)
|
||||
opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(bucket, object)
|
||||
return opts, nil
|
||||
}
|
||||
|
||||
func parseBoolHeader(bucket, object string, h http.Header, headerName string) (bool, error) {
|
||||
value := strings.TrimSpace(h.Get(headerName))
|
||||
if value != "" {
|
||||
switch value {
|
||||
case "true":
|
||||
return true, nil
|
||||
case "false":
|
||||
default:
|
||||
return false, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
Err: fmt.Errorf("Unable to parse %s, value should be either 'true' or 'false'", headerName),
|
||||
}
|
||||
}
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
|
||||
func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
|
||||
opts, err = getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
@@ -180,22 +178,11 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
|
||||
opts.VersionID = nullVersionID
|
||||
}
|
||||
|
||||
delMarker := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceDeleteMarker))
|
||||
if delMarker != "" {
|
||||
switch delMarker {
|
||||
case "true":
|
||||
opts.DeleteMarker = true
|
||||
case "false":
|
||||
default:
|
||||
err = fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceDeleteMarker, fmt.Errorf("DeleteMarker should be true or false"))
|
||||
logger.LogIf(ctx, err)
|
||||
return opts, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
Err: err,
|
||||
}
|
||||
}
|
||||
delMarker, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOSourceDeleteMarker)
|
||||
if err != nil {
|
||||
return opts, err
|
||||
}
|
||||
opts.DeleteMarker = delMarker
|
||||
|
||||
mtime := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime))
|
||||
if mtime != "" {
|
||||
@@ -207,8 +194,6 @@ func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts
|
||||
Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceMTime, err),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
opts.MTime = UTCNow()
|
||||
}
|
||||
return opts, nil
|
||||
}
|
||||
@@ -222,7 +207,6 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
|
||||
if vid != "" && vid != nullVersionID {
|
||||
_, err := uuid.Parse(vid)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
return opts, InvalidVersionID{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
@@ -233,13 +217,13 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
|
||||
return opts, InvalidArgument{
|
||||
Bucket: bucket,
|
||||
Object: object,
|
||||
Err: fmt.Errorf("VersionID specified %s, but versioning not enabled on %s", opts.VersionID, bucket),
|
||||
Err: fmt.Errorf("VersionID specified %s, but versioning not enabled on bucket=%s", opts.VersionID, bucket),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
mtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime))
|
||||
mtime := UTCNow()
|
||||
var mtime time.Time
|
||||
if mtimeStr != "" {
|
||||
mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
|
||||
if err != nil {
|
||||
@@ -251,7 +235,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
|
||||
}
|
||||
}
|
||||
retaintimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceObjectRetentionTimestamp))
|
||||
retaintimestmp := mtime
|
||||
var retaintimestmp time.Time
|
||||
if retaintimeStr != "" {
|
||||
retaintimestmp, err = time.Parse(time.RFC3339, retaintimeStr)
|
||||
if err != nil {
|
||||
@@ -264,7 +248,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
|
||||
}
|
||||
|
||||
lholdtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceObjectLegalHoldTimestamp))
|
||||
lholdtimestmp := mtime
|
||||
var lholdtimestmp time.Time
|
||||
if lholdtimeStr != "" {
|
||||
lholdtimestmp, err = time.Parse(time.RFC3339, lholdtimeStr)
|
||||
if err != nil {
|
||||
@@ -276,7 +260,7 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
|
||||
}
|
||||
}
|
||||
tagtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceTaggingTimestamp))
|
||||
taggingtimestmp := mtime
|
||||
var taggingtimestmp time.Time
|
||||
if tagtimeStr != "" {
|
||||
taggingtimestmp, err = time.Parse(time.RFC3339, tagtimeStr)
|
||||
if err != nil {
|
||||
@@ -367,7 +351,7 @@ func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (O
|
||||
// get ObjectOptions for CompleteMultipart calls
|
||||
func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) {
|
||||
mtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime))
|
||||
mtime := UTCNow()
|
||||
var mtime time.Time
|
||||
if mtimeStr != "" {
|
||||
mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
|
||||
if err != nil {
|
||||
|
||||
@@ -1198,7 +1198,7 @@ func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
|
||||
if disk == nil || disk.Total == 0 {
|
||||
continue
|
||||
}
|
||||
if disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0 {
|
||||
if !globalIsErasureSD && disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0 {
|
||||
// We have an inode count, but not enough inodes.
|
||||
return false, nil
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@ package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
@@ -29,7 +30,6 @@ import (
|
||||
"github.com/minio/minio/internal/event"
|
||||
"github.com/minio/minio/internal/hash"
|
||||
xhttp "github.com/minio/minio/internal/http"
|
||||
"github.com/minio/minio/internal/logger"
|
||||
)
|
||||
|
||||
var etagRegex = regexp.MustCompile("\"*?([^\"]*?)\"*?$")
|
||||
@@ -356,17 +356,8 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
|
||||
PrefixEnabledFn: vc.PrefixEnabled,
|
||||
VersionSuspended: vc.Suspended(),
|
||||
})
|
||||
var logged bool
|
||||
for i, err := range errs {
|
||||
if err != nil {
|
||||
if !logged {
|
||||
// log the first error
|
||||
logger.LogIf(ctx, err)
|
||||
logged = true
|
||||
}
|
||||
continue
|
||||
}
|
||||
dobj := deletedObjs[i]
|
||||
|
||||
for i, dobj := range deletedObjs {
|
||||
oi := ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: dobj.ObjectName,
|
||||
@@ -382,7 +373,7 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
|
||||
oi,
|
||||
ILMExpiry, tags, traceFn)
|
||||
|
||||
sendEvent(eventArgs{
|
||||
evArgs := eventArgs{
|
||||
EventName: event.ObjectRemovedDelete,
|
||||
BucketName: bucket,
|
||||
Object: ObjectInfo{
|
||||
@@ -391,7 +382,13 @@ func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toD
|
||||
},
|
||||
UserAgent: "Internal: [ILM-Expiry]",
|
||||
Host: globalLocalNodeName,
|
||||
})
|
||||
}
|
||||
if errs[i] != nil {
|
||||
evArgs.RespElements = map[string]string{
|
||||
"error": fmt.Sprintf("failed to delete %s(%s), with error %v", dobj.ObjectName, dobj.VersionID, errs[i]),
|
||||
}
|
||||
}
|
||||
sendEvent(evArgs)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+18
-77
@@ -59,7 +59,6 @@ import (
|
||||
"github.com/minio/minio/internal/logger"
|
||||
"github.com/minio/minio/internal/s3select"
|
||||
"github.com/minio/mux"
|
||||
xnet "github.com/minio/pkg/v2/net"
|
||||
"github.com/minio/pkg/v2/policy"
|
||||
)
|
||||
|
||||
@@ -132,9 +131,6 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
// Check for auth type to return S3 compatible error.
|
||||
// type to return the correct error (NoSuchKey vs AccessDenied)
|
||||
@@ -193,9 +189,6 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
||||
defer lock.RUnlock(lkctx)
|
||||
|
||||
getObjectNInfo := objectAPI.GetObjectNInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectNInfo = api.CacheAPI().GetObjectNInfo
|
||||
}
|
||||
|
||||
gopts := opts
|
||||
gopts.NoLock = true // We already have a lock, we can live with it.
|
||||
@@ -252,7 +245,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
HostID: globalDeploymentID(),
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
|
||||
} else {
|
||||
@@ -271,7 +264,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
HostID: globalDeploymentID(),
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
|
||||
} else {
|
||||
@@ -350,9 +343,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
IsOwner: false,
|
||||
}) {
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
_, err = getObjectInfo(ctx, bucket, object, opts)
|
||||
if toAPIError(ctx, err).Code == "NoSuchKey" {
|
||||
@@ -365,9 +355,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
}
|
||||
|
||||
getObjectNInfo := objectAPI.GetObjectNInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectNInfo = api.CacheAPI().GetObjectNInfo
|
||||
}
|
||||
|
||||
// Get request range.
|
||||
var rs *HTTPRangeSpec
|
||||
@@ -547,9 +534,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err), "get-object-handler-write")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -558,9 +542,6 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients
|
||||
logger.LogOnceIf(ctx, fmt.Errorf("Unable to write all the data to client: %w", err), "get-object-handler-close")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@@ -616,9 +597,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
opts, err := getOpts(ctx, r, bucket, object)
|
||||
if err != nil {
|
||||
@@ -650,9 +628,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
||||
IsOwner: false,
|
||||
}) {
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
_, err = getObjectInfo(ctx, bucket, object, opts)
|
||||
if toAPIError(ctx, err).Code == "NoSuchKey" {
|
||||
@@ -780,8 +755,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
|
||||
writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInvalidRange))
|
||||
return
|
||||
}
|
||||
|
||||
logger.LogIf(ctx, err, logger.Application)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1067,7 +1040,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
var srcOpts, dstOpts ObjectOptions
|
||||
srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -1086,16 +1058,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
|
||||
dstOpts, err = copyDstOpts(ctx, r, dstBucket, dstObject, nil)
|
||||
if err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
|
||||
return
|
||||
}
|
||||
cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
|
||||
|
||||
getObjectNInfo := objectAPI.GetObjectNInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectNInfo = api.CacheAPI().GetObjectNInfo
|
||||
}
|
||||
|
||||
checkCopyPrecondFn := func(o ObjectInfo) bool {
|
||||
if _, err := DecryptObjectInfo(&o, r); err != nil {
|
||||
@@ -1378,9 +1346,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction)
|
||||
holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction)
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
// apply default bucket configuration/governance headers for dest side.
|
||||
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms)
|
||||
@@ -1410,7 +1375,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
srcTimestamp := dstOpts.ReplicationSourceLegalholdTimestamp
|
||||
if !srcTimestamp.IsZero() {
|
||||
ondiskTimestamp, err := time.Parse(lastLegalHoldTimestamp, time.RFC3339Nano)
|
||||
// update legalhold metadata only if replica timestamp is newer than what's on disk
|
||||
// update legalhold metadata only if replica timestamp is newer than what's on disk
|
||||
if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) {
|
||||
srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
|
||||
srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcTimestamp.Format(time.RFC3339Nano)
|
||||
@@ -1523,9 +1488,6 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
}
|
||||
|
||||
copyObjectFn := objectAPI.CopyObject
|
||||
if api.CacheAPI() != nil {
|
||||
copyObjectFn = api.CacheAPI().CopyObject
|
||||
}
|
||||
|
||||
// Copy source object to destination, if source and destination
|
||||
// object is same then only metadata is updated.
|
||||
@@ -1572,7 +1534,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
|
||||
objInfo.ETag = origETag
|
||||
enqueueTransitionImmediate(objInfo, lcEventSrc_s3CopyObject)
|
||||
// Remove the transitioned object whose object version is being overwritten.
|
||||
logger.LogIf(ctx, os.Sweep())
|
||||
os.Sweep()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1811,17 +1773,10 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
}
|
||||
}
|
||||
|
||||
if api.CacheAPI() != nil {
|
||||
putObject = api.CacheAPI().PutObject
|
||||
}
|
||||
|
||||
retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction)
|
||||
holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction)
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
|
||||
if s3Err == ErrNone && retentionMode.Valid() {
|
||||
@@ -1972,7 +1927,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
|
||||
// Schedule object for immediate transition if eligible.
|
||||
objInfo.ETag = origETag
|
||||
enqueueTransitionImmediate(objInfo, lcEventSrc_s3PutObject)
|
||||
logger.LogIf(ctx, os.Sweep())
|
||||
os.Sweep()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2115,22 +2070,21 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction)
|
||||
holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction)
|
||||
|
||||
if api.CacheAPI() != nil {
|
||||
putObject = api.CacheAPI().PutObject
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
|
||||
// These are static for all objects extracted.
|
||||
reqParams := extractReqParams(r)
|
||||
respElements := map[string]string{
|
||||
"requestId": w.Header().Get(xhttp.AmzRequestID),
|
||||
"nodeId": w.Header().Get(xhttp.AmzRequestHostID),
|
||||
}
|
||||
if sc == "" {
|
||||
sc = storageclass.STANDARD
|
||||
}
|
||||
|
||||
putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error {
|
||||
size := info.Size()
|
||||
|
||||
if sc == "" {
|
||||
sc = storageclass.STANDARD
|
||||
}
|
||||
|
||||
metadata := map[string]string{
|
||||
xhttp.AmzStorageClass: sc, // save same storage-class as incoming stream.
|
||||
}
|
||||
@@ -2258,8 +2212,8 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
|
||||
EventName: event.ObjectCreatedPut,
|
||||
BucketName: bucket,
|
||||
Object: objInfo,
|
||||
ReqParams: extractReqParams(r),
|
||||
RespElements: extractRespElements(w),
|
||||
ReqParams: reqParams,
|
||||
RespElements: respElements,
|
||||
UserAgent: r.UserAgent(),
|
||||
Host: handlers.GetSourceIP(r),
|
||||
}
|
||||
@@ -2406,9 +2360,6 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
|
||||
})
|
||||
|
||||
deleteObject := objectAPI.DeleteObject
|
||||
if api.CacheAPI() != nil {
|
||||
deleteObject = api.CacheAPI().DeleteObject
|
||||
}
|
||||
|
||||
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
|
||||
objInfo, err := deleteObject(ctx, bucket, object, opts)
|
||||
@@ -2599,9 +2550,6 @@ func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL)
|
||||
@@ -2767,9 +2715,6 @@ func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL)
|
||||
@@ -3067,9 +3012,6 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
|
||||
}
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
// Check for auth type to return S3 compatible error.
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.RestoreObjectAction, bucket, object); s3Error != ErrNone {
|
||||
@@ -3144,7 +3086,6 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
|
||||
}, ObjectOptions{
|
||||
VersionID: objInfo.VersionID,
|
||||
}); err != nil {
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL)
|
||||
return
|
||||
}
|
||||
@@ -3200,7 +3141,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
|
||||
Key: object,
|
||||
Resource: r.URL.Path,
|
||||
RequestID: w.Header().Get(xhttp.AmzRequestID),
|
||||
HostID: globalDeploymentID,
|
||||
HostID: globalDeploymentID(),
|
||||
})
|
||||
writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML)
|
||||
} else {
|
||||
@@ -3224,7 +3165,7 @@ func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *
|
||||
VersionID: objInfo.VersionID,
|
||||
}
|
||||
if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, opts); err != nil {
|
||||
logger.LogIf(ctx, err)
|
||||
logger.LogIf(ctx, fmt.Errorf("Unable to restore transitioned bucket/object %s/%s: %w", bucket, object, err))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -143,14 +143,14 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
|
||||
metadata[xhttp.AmzObjectTagging] = objTags
|
||||
}
|
||||
|
||||
if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
|
||||
metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
|
||||
}
|
||||
retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction)
|
||||
holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction)
|
||||
|
||||
getObjectInfo := objectAPI.GetObjectInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectInfo = api.CacheAPI().GetObjectInfo
|
||||
}
|
||||
|
||||
retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
|
||||
if s3Err == ErrNone && retentionMode.Valid() {
|
||||
@@ -210,9 +210,6 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
|
||||
}
|
||||
|
||||
newMultipartUpload := objectAPI.NewMultipartUpload
|
||||
if api.CacheAPI() != nil {
|
||||
newMultipartUpload = api.CacheAPI().NewMultipartUpload
|
||||
}
|
||||
|
||||
res, err := newMultipartUpload(ctx, bucket, object, opts)
|
||||
if err != nil {
|
||||
@@ -329,9 +326,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
}
|
||||
|
||||
getObjectNInfo := objectAPI.GetObjectNInfo
|
||||
if api.CacheAPI() != nil {
|
||||
getObjectNInfo = api.CacheAPI().GetObjectNInfo
|
||||
}
|
||||
|
||||
// Get request range.
|
||||
var rs *HTTPRangeSpec
|
||||
@@ -356,7 +350,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
return true
|
||||
}
|
||||
if parseRangeErr != nil {
|
||||
logger.LogIf(ctx, parseRangeErr)
|
||||
writeCopyPartErr(ctx, w, parseRangeErr, r.URL)
|
||||
// Range header mismatch is pre-condition like failure
|
||||
// so return true to indicate Range precondition failed.
|
||||
@@ -543,9 +536,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
|
||||
|
||||
srcInfo.PutObjReader = pReader
|
||||
copyObjectPart := objectAPI.CopyObjectPart
|
||||
if api.CacheAPI() != nil {
|
||||
copyObjectPart = api.CacheAPI().CopyObjectPart
|
||||
}
|
||||
|
||||
// Copy source object to destination, if source and destination
|
||||
// object is same then only metadata is updated.
|
||||
partInfo, err := copyObjectPart(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID,
|
||||
@@ -822,9 +813,6 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
|
||||
opts.IndexCB = idxCb
|
||||
|
||||
putObjectPart := objectAPI.PutObjectPart
|
||||
if api.CacheAPI() != nil {
|
||||
putObjectPart = api.CacheAPI().PutObjectPart
|
||||
}
|
||||
|
||||
partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, pReader, opts)
|
||||
if err != nil {
|
||||
@@ -935,9 +923,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
}
|
||||
|
||||
completeMultiPartUpload := objectAPI.CompleteMultipartUpload
|
||||
if api.CacheAPI() != nil {
|
||||
completeMultiPartUpload = api.CacheAPI().CompleteMultipartUpload
|
||||
}
|
||||
|
||||
versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
|
||||
suspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
|
||||
@@ -1035,7 +1020,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
|
||||
if !globalTierConfigMgr.Empty() {
|
||||
// Schedule object for immediate transition if eligible.
|
||||
enqueueTransitionImmediate(objInfo, lcEventSrc_s3CompleteMultipartUpload)
|
||||
logger.LogIf(ctx, os.Sweep())
|
||||
os.Sweep()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1059,9 +1044,6 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
|
||||
return
|
||||
}
|
||||
abortMultipartUpload := objectAPI.AbortMultipartUpload
|
||||
if api.CacheAPI() != nil {
|
||||
abortMultipartUpload = api.CacheAPI().AbortMultipartUpload
|
||||
}
|
||||
|
||||
if s3Error := checkRequestAuthType(ctx, r, policy.AbortMultipartUploadAction, bucket, object); s3Error != ErrNone {
|
||||
writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
|
||||
|
||||
@@ -105,7 +105,8 @@ func osTrace(s osMetric, startTime time.Time, duration time.Duration, path strin
|
||||
|
||||
func updateOSMetrics(s osMetric, paths ...string) func(err error) {
|
||||
if globalTrace.NumSubscribers(madmin.TraceOS) == 0 {
|
||||
return func(err error) { globalOSMetrics.time(s) }
|
||||
osAction := globalOSMetrics.time(s)
|
||||
return func(err error) { osAction() }
|
||||
}
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
+4
-4
@@ -80,11 +80,11 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
|
||||
continue
|
||||
}
|
||||
|
||||
var typ os.FileMode = 0 // regular file
|
||||
var typ os.FileMode // regular file
|
||||
switch {
|
||||
case data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0:
|
||||
// Reparse point is a symlink
|
||||
fi, err := os.Stat(pathJoin(dirPath, string(name)))
|
||||
fi, err := os.Stat(pathJoin(dirPath, name))
|
||||
if err != nil {
|
||||
// It got deleted in the meantime, not found
|
||||
// or returns too many symlinks ignore this
|
||||
@@ -153,7 +153,7 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
|
||||
switch {
|
||||
case data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0:
|
||||
// Reparse point is a symlink
|
||||
fi, err := os.Stat(pathJoin(dirPath, string(name)))
|
||||
fi, err := os.Stat(pathJoin(dirPath, name))
|
||||
if err != nil {
|
||||
// It got deleted in the meantime, not found
|
||||
// or returns too many symlinks ignore this
|
||||
@@ -170,7 +170,7 @@ func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err er
|
||||
continue
|
||||
}
|
||||
case data.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0:
|
||||
name = name + SlashSeparator
|
||||
name += SlashSeparator
|
||||
}
|
||||
|
||||
count--
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user